Miscelaneous perf commits

lib/dead_end/clean_document.rb

@@ -85,8 +85,8 @@ module DeadEnd
   #
   class CleanDocument
     def initialize(source:)
-      @source = clean_sweep(source: source)
-      @document = CodeLine.from_source(@source)
+      lines = clean_sweep(source: source)
+      @document = CodeLine.from_source(lines.join, lines: lines)
     end
 
     # Call all of the document "cleaners"
@@ -161,7 +161,7 @@ def clean_sweep(source:)
         else
           line
         end
-      end.join
+      end
     end
 
     # Smushes all heredoc lines into one line

lib/dead_end/code_line.rb

@@ -26,9 +26,10 @@ class CodeLine
 
     # Returns an array of CodeLine objects
     # from the source string
-    def self.from_source(source)
-      lex_array_for_line = LexAll.new(source: source).each_with_object(Hash.new { |h, k| h[k] = [] }) { |lex, hash| hash[lex.line] << lex }
-      source.lines.map.with_index do |line, index|
+    def self.from_source(source, lines: nil)
+      lines ||= source.lines
+      lex_array_for_line = LexAll.new(source: source, source_lines: lines).each_with_object(Hash.new { |h, k| h[k] = [] }) { |lex, hash| hash[lex.line] << lex }
+      lines.map.with_index do |line, index|
         CodeLine.new(
           line: line,
           index: index,
@@ -42,28 +43,20 @@ def initialize(line:, index:, lex:)
       @lex = lex
       @line = line
       @index = index
-      @original = line.freeze
+      @original = line
       @line_number = @index + 1
+      strip_line = line.dup
+      strip_line.lstrip!
 
-      if line.strip.empty?
+      if strip_line.empty?
         @empty = true
         @indent = 0
       else
         @empty = false
-        @indent = SpaceCount.indent(line)
+        @indent = line.length - strip_line.length
       end
 
-      kw_count = 0
-      end_count = 0
-      @lex.each do |lex|
-        kw_count += 1 if lex.is_kw?
-        end_count += 1 if lex.is_end?
-      end
-
-      kw_count -= oneliner_method_count
-
-      @is_kw = (kw_count - end_count) > 0
-      @is_end = (end_count - kw_count) > 0
+      set_kw_end
     end
 
     # Used for stable sort via indentation level
@@ -179,8 +172,7 @@ def <=>(other)
     #
     # For some reason this introduces `on_ignore_newline` but with BEG type
     def ignore_newline_not_beg?
-      lex_value = lex.detect { |l| l.type == :on_ignored_nl }
-      !!(lex_value && !lex_value.expr_beg?)
+      @ignore_newline_not_beg
     end
 
     # Determines if the given line has a trailing slash
@@ -206,11 +198,22 @@ def trailing_slash?
     #
     #   ENDFN -> BEG (token = '=' ) -> END
     #
-    private def oneliner_method_count
+    private def set_kw_end
       oneliner_count = 0
       in_oneliner_def = nil
 
+      kw_count = 0
+      end_count = 0
+
+      @ignore_newline_not_beg = false
       @lex.each do |lex|
+        kw_count += 1 if lex.is_kw?
+        end_count += 1 if lex.is_end?
+
+        if lex.type == :on_ignored_nl
+          @ignore_newline_not_beg = !lex.expr_beg?
+        end
+
         if in_oneliner_def.nil?
           in_oneliner_def = :ENDFN if lex.state.allbits?(Ripper::EXPR_ENDFN)
         elsif lex.state.allbits?(Ripper::EXPR_ENDFN)
@@ -227,7 +230,10 @@ def trailing_slash?
         end
       end
 
-      oneliner_count
+      kw_count -= oneliner_count
+
+      @is_kw = (kw_count - end_count) > 0
+      @is_end = (end_count - kw_count) > 0
     end
   end
 end

lib/dead_end/lex_all.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module DeadEnd
   # Ripper.lex is not guaranteed to lex the entire source document
   #
@@ -8,20 +10,22 @@ module DeadEnd
   class LexAll
     include Enumerable
 
-    def initialize(source:)
-      @lex = Ripper.lex(source)
-      lineno = @lex.last.first.first + 1
-      source_lines = source.lines
-      last_lineno = source_lines.count
+    def initialize(source:, source_lines: nil)
+      @lex = Ripper::Lexer.new(source, "-", 1).parse.sort_by(&:pos)
+      lineno = @lex.last.pos.first + 1
+      source_lines ||= source.lines
+      last_lineno = source_lines.length
 
       until lineno >= last_lineno
         lines = source_lines[lineno..-1]
 
-        @lex.concat(Ripper.lex(lines.join, "-", lineno + 1))
-        lineno = @lex.last.first.first + 1
+        @lex.concat(
+          Ripper::Lexer.new(lines.join, "-", lineno + 1).parse.sort_by(&:pos)
+        )
+        lineno = @lex.last.pos.first + 1
       end
 
-      @lex.map! { |(line, _), type, token, state| LexValue.new(line, type, token, state) }
+      @lex.map! { |elem| LexValue.new(elem.pos.first, elem.event, elem.tok, elem.state) }
     end
 
     def to_a

lib/dead_end/lex_value.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module DeadEnd
   # Value object for accessing lex values
   #