Handle advancing with possibile sub-character end.

When searching in a substring with an endIndex that isn't on a character boundary,
advancing from the penultimate character position to the substring's endIndex can end
up failing, since advancing by a character finds the character-aligned index that is
_past_ the substring end. This change handles that case.

Author: natecook1000

Sources/_StringProcessing/ConsumerInterface.swift

@@ -24,47 +24,6 @@ extension DSLTree._AST.Atom {
   }
 }
 
-extension Character {
-//  func generateConsumer(
-//    _ opts: MatchingOptions
-//  ) throws -> MEProgram.ConsumeFunction {
-//    let isCaseInsensitive = opts.isCaseInsensitive
-//    switch opts.semanticLevel {
-//    case .graphemeCluster:
-//      return { input, bounds in
-//        guard let (char, next) = input.characterAndEnd(
-//          at: bounds.lowerBound, limitedBy: bounds.upperBound)
-//        else { return nil }
-//        if isCaseInsensitive && isCased {
-//          return char.lowercased() == self.lowercased()
-//            ? next
-//            : nil
-//        } else {
-//          return char == self
-//            ? next
-//            : nil
-//        }
-//      }
-//    case .unicodeScalar:
-//      // TODO: This should only be reachable from character class emission, can
-//      // we guarantee that? Otherwise we'd want a different matching behavior.
-//      let consumers = unicodeScalars.map { s in consumeScalar {
-//        isCaseInsensitive
-//          ? $0.properties.lowercaseMapping == s.properties.lowercaseMapping
-//          : $0 == s
-//      }}
-//      return { input, bounds in
-//        for fn in consumers {
-//          if let idx = fn(input, bounds) {
-//            return idx
-//          }
-//        }
-//        return nil
-//      }
-//    }
-//  }
-}
-
 extension DSLTree.Atom {
   var singleScalarASCIIValue: UInt8? {
     switch self {
@@ -78,76 +37,6 @@ extension DSLTree.Atom {
       return nil
     }
   }
-  
-  // TODO: If ByteCodeGen switches first, then this is unnecessary for
-  // top-level nodes, but it's also invoked for `.atom` members of a custom CC
-//  func generateConsumer(
-//    _ opts: MatchingOptions
-//  ) throws -> MEProgram.ConsumeFunction? {
-//    switch self {
-//    case let .char(c):
-//      return try c.generateConsumer(opts)
-//
-//    case let .scalar(s):
-//      // A scalar always matches the same as a single scalar character. This
-//      // means it must match a whole grapheme in grapheme semantic mode, but
-//      // can match a single scalar in scalar semantic mode.
-//      return try Character(s).generateConsumer(opts)
-//
-//    case .any:
-//      // FIXME: Should this be a total ordering?
-//      if opts.semanticLevel == .graphemeCluster {
-//        return { input, bounds in
-//          input.index(after: bounds.lowerBound)
-//        }
-//      } else {
-//        return consumeScalar { _ in
-//          true
-//        }
-//      }
-//
-//    case .anyNonNewline:
-//      switch opts.semanticLevel {
-//      case .graphemeCluster:
-//        return { input, bounds in
-//          input[bounds.lowerBound].isNewline
-//            ? nil
-//            : input.index(after: bounds.lowerBound)
-//        }
-//      case .unicodeScalar:
-//        return { input, bounds in
-//          input[bounds.lowerBound].isNewline
-//            ? nil
-//            : input.unicodeScalars.index(after: bounds.lowerBound)
-//        }
-//      }
-//
-//    case .dot:
-//      throw Unreachable(".atom(.dot) should be handled by emitDot")
-//
-//    case .assertion:
-//      // TODO: We could handle, should this be total?
-//      return nil
-//    case .characterClass(let cc):
-//      return cc.generateConsumer(opts)
-//
-//    case .backreference:
-//      // TODO: Should we handle?
-//      return nil
-//
-//    case .symbolicReference:
-//      // TODO: Should we handle?
-//      return nil
-//
-//    case .changeMatchingOptions:
-//      // TODO: Should we handle?
-//      return nil
-//
-//    case let .unconverted(a):
-//      return try a.ast.generateConsumer(opts)
-//    }
-//
-//  }
 }
 
 extension DSLTree.Atom.CharacterClass {
@@ -389,28 +278,6 @@ extension DSLTree.CustomCharacterClass {
       }
     )
   }
-  
-//  func generateConsumer(
-//    _ opts: MatchingOptions
-//  ) throws -> MEProgram.ConsumeFunction {
-//    // NOTE: Easy way to implement, obviously not performant
-//    let consumers = try members.map {
-//      try $0.generateConsumer(opts)
-//    }
-//    return { input, bounds in
-//      for consumer in consumers {
-//        if let idx = consumer(input, bounds) {
-//          return isInverted ? nil : idx
-//        }
-//      }
-//      if isInverted {
-//        return opts.semanticLevel == .graphemeCluster
-//          ? Swift.min(input.index(after: bounds.lowerBound), bounds.upperBound)
-//          : input.unicodeScalars.index(after: bounds.lowerBound)
-//      }
-//      return nil
-//    }
-//  }
 }
 
 // NOTE: Conveniences, though not most performant
@@ -519,7 +386,6 @@ extension AST.Atom.CharacterProperty {
 
       case .generalCategory(let p):
         return try p.generateConsumer(opts)
-//        fatalError("TODO: Map categories: \(p)")
 
       case .binary(let prop, value: let value):
         let cons = try prop.generateConsumer(opts)

Sources/_StringProcessing/Engine/Processor.swift

@@ -175,14 +175,25 @@ extension Processor {
   // save point was restored
   mutating func consume(_ n: Distance) -> Bool {
     // TODO: needs benchmark coverage
-    guard let idx = input.index(
+    if let idx = input.index(
       currentPosition, offsetBy: n.rawValue, limitedBy: end
-    ) else {
-      signalFailure()
-      return false
+    ) {
+      currentPosition = idx
+      return true
     }
-    currentPosition = idx
-    return true
+
+    // If `end` falls in the middle of a character, and we are trying to advance
+    // by one "character", then we should max out at `end` even though the above
+    // advancement will result in `nil`.
+    if n == 1, let idx = input.unicodeScalars.index(
+      currentPosition, offsetBy: n.rawValue, limitedBy: end
+    ) {
+      currentPosition = idx
+      return true
+    }
+
+    signalFailure()
+    return false
   }
 
   // Advances in unicode scalar view