Use raw Lexer::Elem

Related to ruby/ruby#5093

Instead of building a bunch of objects then converting them into arrays, we can natively use the objects.

Total allocated: 25407256 bytes (261284 objects)
Total retained:  1937110 bytes (30266 objects)

allocated memory by location
-----------------------------------
  10045332  <internal:kernel>:90
   5273752  Documents/projects/dead_end/lib/dead_end/code_frontier.rb:131
   2267256  Documents/projects/dead_end/lib/dead_end.rb:65
    986760  .rubies/ruby-3.0.2/lib/ruby/3.0.0/ripper/lexer.rb:189
    938301  .rubies/ruby-3.0.2/lib/ruby/3.0.0/ripper/lexer.rb:132
    734800  Documents/projects/dead_end/lib/dead_end/lex_all.rb:24
    668000  .rubies/ruby-3.0.2/lib/ruby/3.0.0/ripper/lexer.rb:97    <== HERE
    471980  Documents/projects/dead_end/lib/dead_end.rb:97
    334040  .rubies/ruby-3.0.2/lib/ruby/3.0.0/ripper/lexer.rb:58
    334040  .rubies/ruby-3.0.2/lib/ruby/3.0.0/ripper/lexer.rb:72

After

Total allocated: 19648672 bytes (252933 objects)
Total retained:  1944374 bytes (30266 objects)

allocated memory by location
-----------------------------------
  10045332  <internal:kernel>:90
   5273752  Documents/projects/dead_end/lib/dead_end/code_frontier.rb:131
   2267256  Documents/projects/dead_end/lib/dead_end.rb:65
    986760  .rubies/ruby-3.0.2/lib/ruby/3.0.0/ripper/lexer.rb:189
    938301  .rubies/ruby-3.0.2/lib/ruby/3.0.0/ripper/lexer.rb:132
    734800  Documents/projects/dead_end/lib/dead_end/lex_all.rb:26

    471980  Documents/projects/dead_end/lib/dead_end.rb:97
    334040  .rubies/ruby-3.0.2/lib/ruby/3.0.0/ripper/lexer.rb:58
    334040  .rubies/ruby-3.0.2/lib/ruby/3.0.0/ripper/lexer.rb:72
    329051  Documents/projects/dead_end/lib/dead_end/code_line.rb:31
    218144  Documents/projects/dead_end/lib/dead_end/code_line.rb:30

Author: schneems

lib/dead_end/lex_all.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module DeadEnd
   # Ripper.lex is not guaranteed to lex the entire source document
   #
@@ -9,19 +11,21 @@ class LexAll
     include Enumerable
 
     def initialize(source:, source_lines: nil)
-      @lex = Ripper.lex(source)
-      lineno = @lex.last.first.first + 1
+      @lex = Ripper::Lexer.new(source, "-", 1).parse.sort_by(&:pos)
+      lineno = @lex.last.pos.first + 1
       source_lines ||= source.lines
       last_lineno = source_lines.length
 
       until lineno >= last_lineno
         lines = source_lines[lineno..-1]
 
-        @lex.concat(Ripper.lex(lines.join, "-", lineno + 1))
-        lineno = @lex.last.first.first + 1
+        @lex.concat(
+          Ripper::Lexer.new(lines.join, "-", lineno + 1).parse.sort_by(&:pos)
+        )
+        lineno = @lex.last.pos.first + 1
       end
 
-      @lex.map! { |(line, _), type, token, state| LexValue.new(line, type, token, state) }
+      @lex.map! { |elem| LexValue.new(elem.pos.first, elem.event, elem.tok, elem.state) }
     end
 
     def to_a

lib/dead_end/lex_value.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module DeadEnd
   # Value object for accessing lex values
   #