Lex whole source document

[schneems]

For some weird reason Ripper.lex doesn't lex the whole source:

source = <<~EOM
  describe "cat" # 1
    Cat.call do  # 2
    end          # 3
  end            # 4
                 # 5
  it "dog" do    # 6 
    Dog.call do  # 7
    end          # 8
  end            # 9
EOM

require 'ripper'
pp Ripper.lex(source)
# => 
#    [[[1, 0], :on_ident, "describe", CMDARG],
#     [[1, 8], :on_sp, " ", CMDARG],
#     [[1, 9], :on_tstring_beg, "\"", CMDARG],
#     [[1, 10], :on_tstring_content, "cat", CMDARG],
#     [[1, 13], :on_tstring_end, "\"", END],
#     [[1, 14], :on_sp, " ", END],
#     [[1, 15], :on_comment, "# 1\n", END],
#     [[2, 0], :on_sp, "  ", BEG],
#     [[2, 2], :on_const, "Cat", CMDARG],
#     [[2, 5], :on_period, ".", DOT],
#     [[2, 6], :on_ident, "call", ARG],
#     [[2, 10], :on_sp, " ", ARG],
#     [[2, 11], :on_kw, "do", BEG],
#     [[2, 13], :on_sp, "  ", BEG],
#     [[2, 15], :on_comment, "# 2\n", BEG],
#     [[3, 0], :on_sp, "  ", BEG],
#     [[3, 2], :on_kw, "end", END],
#     [[3, 5], :on_sp, "          ", END],
#     [[3, 15], :on_comment, "# 3\n", END],
#     [[4, 0], :on_kw, "end", END]]#    

Note that there's no "dog" in this lex even though it's in our source. To handle this I'm introducing a new class LexAll. It also wraps lex values in a easier to use value object.