Use caching in oniguruma

benchmark/benchmark.coffee

@@ -1,6 +1,6 @@
 path = require 'path'
 fs = require 'fs-plus'
-GrammarRegistry = require '../lib/grammar-registry'
+GrammarRegistry = require '../src/grammar-registry'
 
 registry = new GrammarRegistry()
 jsGrammar = registry.loadGrammarSync(path.resolve(__dirname, '..', 'spec', 'fixtures', 'javascript.json'))
@@ -10,10 +10,11 @@ cssGrammar.maxTokensPerLine = Infinity
 
 tokenize = (grammar, content, lineCount) ->
   start = Date.now()
-  {tags} = grammar.tokenizeLines(content)
+  tokenizedLines = grammar.tokenizeLines(content, false)
   duration = Date.now() - start
   tokenCount = 0
-  tokenCount++ for tag in tags when tag >= 0
+  for tokenizedLine in tokenizedLines
+    tokenCount += tokenizedLine.length
   tokensPerMillisecond = Math.round(tokenCount / duration)
   console.log "Generated #{tokenCount} tokens for #{lineCount} lines in #{duration}ms (#{tokensPerMillisecond} tokens/ms)"
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "first-mate",
-  "version": "7.0.2",
+  "version": "7.0.3-2",
   "description": "TextMate helpers",
   "main": "./lib/first-mate.js",
   "scripts": {
@@ -22,7 +22,7 @@
     "event-kit": "^2.2.0",
     "fs-plus": "^3.0.0",
     "grim": "^2.0.1",
-    "oniguruma": "^6.1.0",
+    "oniguruma": "6.2.0",
     "season": "^6.0.0",
     "underscore-plus": "^1"
   },

spec/grammar-registry-spec.coffee

@@ -1,5 +1,5 @@
 path = require 'path'
-GrammarRegistry = require '../lib/grammar-registry'
+GrammarRegistry = require '../src/grammar-registry'
 
 describe "GrammarRegistry", ->
   registry = null

spec/grammar-spec.coffee

@@ -1,8 +1,8 @@
 path = require 'path'
 _ = require 'underscore-plus'
 fs = require 'fs-plus'
-GrammarRegistry = require '../lib/grammar-registry'
-Grammar = require '../lib/grammar'
+GrammarRegistry = require '../src/grammar-registry'
+Grammar = require '../src/grammar'
 
 describe "Grammar tokenization", ->
   [grammar, registry] = []

src/grammar.coffee

@@ -2,7 +2,7 @@ path = require 'path'
 
 _ = require 'underscore-plus'
 fs = require 'fs-plus'
-{OnigRegExp} = require 'oniguruma'
+{OnigRegExp, OnigString} = require 'oniguruma'
 {Emitter} = require 'event-kit'
 Grim = require 'grim'
 
@@ -68,13 +68,13 @@ class Grammar
   # * `text` A {String} containing one or more lines.
   #
   # Returns an {Array} of token arrays for each line tokenized.
-  tokenizeLines: (text) ->
+  tokenizeLines: (text, compatibilityMode=true) ->
     lines = text.split('\n')
     ruleStack = null
 
     scopes = []
     for line, lineNumber in lines
-      {tags, ruleStack} = @tokenizeLine(line, ruleStack, lineNumber is 0)
+      {tags, ruleStack} = @tokenizeLine(line, ruleStack, lineNumber is 0, compatibilityMode)
       @registry.decodeTokens(line, tags, scopes)
 
   # Public: Tokenize the line of text.
@@ -108,6 +108,8 @@ class Grammar
     else
       line = inputLine
 
+    string = new OnigString(line + '\n')
+
     if ruleStack?
       ruleStack = ruleStack.slice()
       if compatibilityMode
@@ -137,7 +139,7 @@ class Grammar
         truncatedLine = true
         break
 
-      if match = _.last(ruleStack).rule.getNextTags(ruleStack, line, position, firstLine)
+      if match = _.last(ruleStack).rule.getNextTags(ruleStack, string, position, firstLine)
         {nextTags, tagsStart, tagsEnd} = match
 
         # Unmatched text before next tags

src/pattern.coffee

@@ -83,7 +83,7 @@ class Pattern
     beginCaptures = []
 
     for {start, end} in beginCaptureIndices
-      beginCaptures.push(line[start...end])
+      beginCaptures.push(line.substring(start, end))
 
     resolvedMatch = @match.replace AllDigitsRegex, (match) ->
       index = parseInt(match[1..])

src/rule.coffee

@@ -52,8 +52,7 @@ class Rule
       capture.start = Math.min(capture.start, lineLength)
     return
 
-  findNextMatch: (ruleStack, line, position, firstLine) ->
-    lineWithNewline = "#{line}\n"
+  findNextMatch: (ruleStack, lineWithNewline, position, firstLine) ->
     baseGrammar = ruleStack[0].rule.grammar
     results = []
 

src/scope-selector.coffee

@@ -1,4 +1,4 @@
-ScopeSelectorParser = require './scope-selector-parser'
+ScopeSelectorParser = require '../lib/scope-selector-parser'
 
 module.exports =
 class ScopeSelector