More unicode support

Author: som-snytt

compiler/src/dotty/tools/dotc/core/NameOps.scala

@@ -86,11 +86,17 @@ object NameOps {
     def isVarPattern: Boolean =
       testSimple { n =>
         n.length > 0 && {
+          def isLowerLetterSupplementary: Boolean =
+            import Character.{isHighSurrogate, isLowSurrogate, isLetter, isLowerCase, isValidCodePoint, toCodePoint}
+            isHighSurrogate(n(0)) && n.length > 1 && isLowSurrogate(n(1)) && {
+              val codepoint = toCodePoint(n(0), n(1))
+              isValidCodePoint(codepoint) && isLetter(codepoint) && isLowerCase(codepoint)
+            }
           val first = n.head
-          (((first.isLower && first.isLetter) || first == '_')
-            && (n != false_)
-            && (n != true_)
-            && (n != null_))
+          ((first.isLower && first.isLetter || first == '_' || isLowerLetterSupplementary)
+            && n != false_
+            && n != true_
+            && n != null_)
         }
       } || name.is(PatMatGivenVarName)
 

compiler/src/dotty/tools/dotc/core/Names.scala

@@ -15,8 +15,8 @@ import scala.annotation.internal.sharable
 object Names {
   import NameKinds._
 
-  /** Things that can be turned into names with `totermName` and `toTypeName`
-   *  Decorators defines implements these as extension methods for strings.
+  /** Things that can be turned into names with `toTermName` and `toTypeName`
+   *  Decorators implements these as extension methods for strings.
    */
   type PreName = Name | String
 
@@ -25,7 +25,7 @@ object Names {
    */
   abstract class Designator
 
-  /** A name if either a term name or a type name. Term names can be simple
+  /** A name is either a term name or a type name. Term names can be simple
    *  or derived. A simple term name is essentially an interned string stored
    *  in a name table. A derived term name adds a tag, and possibly a number
    *  or a further simple name to some other name.

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

@@ -21,6 +21,8 @@ import config.Feature.migrateTo3
 import config.SourceVersion.`3.0`
 import reporting.{NoProfile, Profile}
 
+import java.util.Objects
+
 object Scanners {
 
   /** Offset into source character array */
@@ -792,19 +794,21 @@ object Scanners {
     private def isSupplementary(high: Char, test: Int => Boolean, strict: Boolean = true): Boolean =
       isHighSurrogate(high) && {
         var res = false
-        nextChar()
-        val low = ch
+        val low = lookaheadChar()
         if isLowSurrogate(low) then
-          nextChar()
           val codepoint = toCodePoint(high, low)
-          if isValidCodePoint(codepoint) && test(codepoint) then
-            putChar(high)
-            putChar(low)
-            res = true
+          if isValidCodePoint(codepoint) then
+            if test(codepoint) then
+              putChar(high)
+              putChar(low)
+              nextChar()
+              nextChar()
+              res = true
           else
             error(s"illegal character '${toUnicode(high)}${toUnicode(low)}'")
         else if !strict then
           putChar(high)
+          nextChar()
           res = true
         else
           error(s"illegal character '${toUnicode(high)}' missing low surrogate")
@@ -904,7 +908,6 @@ object Scanners {
               if (ch == '\"') {
                 if (lookaheadChar() == '\"') {
                   nextRawChar()
-                  //offset += 3   // first part is positioned at the quote
                   nextRawChar()
                   stringPart(multiLine = true)
                 }
@@ -915,7 +918,6 @@ object Scanners {
                 }
               }
               else {
-                //offset += 1   // first part is positioned at the quote
                 stringPart(multiLine = false)
               }
             }
@@ -1000,10 +1002,11 @@ object Scanners {
               nextChar(); token = LARROW
               report.deprecationWarning("The unicode arrow `←` is deprecated, use `<-` instead. If you still wish to display it as one character, consider using a font with programming ligatures such as Fira Code.", sourcePos(offset))
             }
-            else if (Character.isUnicodeIdentifierStart(ch)) {
+            else if (isUnicodeIdentifierStart(ch)) {
               putChar(ch)
               nextChar()
               getIdentRest()
+              if (ch == '"' && token == IDENTIFIER) token = INTERPOLATIONID
             }
             else if (isSpecial(ch)) {
               putChar(ch)
@@ -1012,6 +1015,9 @@ object Scanners {
             }
             else if isSupplementary(ch, isUnicodeIdentifierStart) then
               getIdentRest()
+              if (ch == '"' && token == IDENTIFIER) token = INTERPOLATIONID
+            else if isSupplementary(ch, isSpecial) then
+              getOperatorRest()
             else {
               error(s"illegal character '${toUnicode(ch)}'")
               nextChar()
@@ -1130,7 +1136,7 @@ object Scanners {
       else error("unclosed quoted identifier")
     }
 
-    private def getIdentRest(): Unit = (ch: @switch) match {
+    @tailrec private def getIdentRest(): Unit = (ch: @switch) match {
       case 'A' | 'B' | 'C' | 'D' | 'E' |
            'F' | 'G' | 'H' | 'I' | 'J' |
            'K' | 'L' | 'M' | 'N' | 'O' |
@@ -1165,7 +1171,7 @@ object Scanners {
           finishNamed()
     }
 
-    private def getOperatorRest(): Unit = (ch: @switch) match {
+    @tailrec private def getOperatorRest(): Unit = (ch: @switch) match {
       case '~' | '!' | '@' | '#' | '%' |
            '^' | '*' | '+' | '-' | '<' |
            '>' | '?' | ':' | '=' | '&' |
@@ -1177,22 +1183,12 @@ object Scanners {
         else { putChar(ch); nextChar(); getOperatorRest() }
       case _ =>
         if (isSpecial(ch)) { putChar(ch); nextChar(); getOperatorRest() }
+        else if (isSupplementary(ch, isSpecial)) getOperatorRest()
         else finishNamed()
     }
 
     private def getIdentOrOperatorRest(): Unit =
-      if (isIdentifierPart(ch))
-        getIdentRest()
-      else ch match {
-        case '~' | '!' | '@' | '#' | '%' |
-             '^' | '*' | '+' | '-' | '<' |
-             '>' | '?' | ':' | '=' | '&' |
-             '|' | '\\' | '/' =>
-          getOperatorRest()
-        case _ =>
-          if (isSpecial(ch)) getOperatorRest()
-          else finishNamed()
-      }
+      if (isIdentifierPart(ch) || isSupplementary(ch, isIdentifierPart)) getIdentRest() else getOperatorRest()
 
     def isSoftModifier: Boolean =
       token == IDENTIFIER
@@ -1515,7 +1511,7 @@ object Scanners {
       if (ch == '\'') finishCharLit()
       else {
         token = op
-        strVal = if (name != null) name.toString else null
+        strVal = Objects.toString(name)
         litBuf.clear()
       }
     }

compiler/src/dotty/tools/dotc/util/Chars.scala

@@ -1,21 +1,20 @@
 package dotty.tools.dotc.util
 
 import scala.annotation.switch
-import java.lang.{Character => JCharacter}
-import java.lang.Character.LETTER_NUMBER
-import java.lang.Character.LOWERCASE_LETTER
-import java.lang.Character.OTHER_LETTER
-import java.lang.Character.TITLECASE_LETTER
-import java.lang.Character.UPPERCASE_LETTER
+import Character.{LETTER_NUMBER, LOWERCASE_LETTER, OTHER_LETTER, TITLECASE_LETTER, UPPERCASE_LETTER}
+import Character.{MATH_SYMBOL, OTHER_SYMBOL}
+import Character.{isJavaIdentifierPart, isUnicodeIdentifierStart, isUnicodeIdentifierPart}
 
 /** Contains constants and classifier methods for characters */
-object Chars {
+object Chars:
 
   inline val LF = '\u000A'
   inline val FF = '\u000C'
   inline val CR = '\u000D'
   inline val SU = '\u001A'
 
+  type CodePoint = Int
+
   /** Convert a character digit to an Int according to given base,
     *  -1 if no success
     */
@@ -59,17 +58,21 @@ object Chars {
     '0' <= c && c <= '9' || 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z'
 
   /** Can character start an alphanumeric Scala identifier? */
-  def isIdentifierStart(c: Char): Boolean =
-    (c == '_') || (c == '$') || JCharacter.isUnicodeIdentifierStart(c)
+  def isIdentifierStart(c: Char): Boolean = (c == '_') || (c == '$') || isUnicodeIdentifierStart(c)
+  def isIdentifierStart(c: CodePoint): Boolean = (c == '_') || (c == '$') || isUnicodeIdentifierStart(c)
 
   /** Can character form part of an alphanumeric Scala identifier? */
-  def isIdentifierPart(c: Char): Boolean =
-    (c == '$') || JCharacter.isUnicodeIdentifierPart(c)
+  def isIdentifierPart(c: Char): Boolean = (c == '$') || isUnicodeIdentifierPart(c)
+  def isIdentifierPart(c: CodePoint) = (c == '$') || isUnicodeIdentifierPart(c)
 
   /** Is character a math or other symbol in Unicode?  */
   def isSpecial(c: Char): Boolean = {
-    val chtp = JCharacter.getType(c)
-    chtp == JCharacter.MATH_SYMBOL.toInt || chtp == JCharacter.OTHER_SYMBOL.toInt
+    val chtp = Character.getType(c)
+    chtp == MATH_SYMBOL.toInt || chtp == OTHER_SYMBOL.toInt
+  }
+  def isSpecial(codePoint: CodePoint) = {
+    val chtp = Character.getType(codePoint)
+    chtp == MATH_SYMBOL.toInt || chtp == OTHER_SYMBOL.toInt
   }
 
   def isValidJVMChar(c: Char): Boolean =
@@ -78,15 +81,19 @@ object Chars {
   def isValidJVMMethodChar(c: Char): Boolean =
     !(c == '.' || c == ';' || c =='[' || c == '/' || c == '<' || c == '>')
 
-  private final val otherLetters = Set[Char]('\u0024', '\u005F')  // '$' and '_'
-  private final val letterGroups = {
-    import JCharacter._
-    Set[Byte](LOWERCASE_LETTER, UPPERCASE_LETTER, OTHER_LETTER, TITLECASE_LETTER, LETTER_NUMBER)
-  }
-  def isScalaLetter(ch: Char): Boolean = letterGroups(JCharacter.getType(ch).toByte) || otherLetters(ch)
+  def isScalaLetter(c: Char): Boolean =
+    Character.getType(c: @switch) match {
+      case LOWERCASE_LETTER | UPPERCASE_LETTER | OTHER_LETTER | TITLECASE_LETTER | LETTER_NUMBER => true
+      case _ => c == '$' || c == '_'
+    }
+  def isScalaLetter(c: CodePoint): Boolean =
+    Character.getType(c: @switch) match {
+      case LOWERCASE_LETTER | UPPERCASE_LETTER | OTHER_LETTER | TITLECASE_LETTER | LETTER_NUMBER => true
+      case _ => c == '$' || c == '_'
+    }
 
   /** Can character form part of a Scala operator name? */
-  def isOperatorPart(c : Char) : Boolean = (c: @switch) match {
+  def isOperatorPart(c: Char): Boolean = (c: @switch) match {
     case '~' | '!' | '@' | '#' | '%' |
          '^' | '*' | '+' | '-' | '<' |
          '>' | '?' | ':' | '=' | '&' |
@@ -95,5 +102,4 @@ object Chars {
   }
 
   /** Would the character be encoded by `NameTransformer.encode`? */
-  def willBeEncoded(c : Char) : Boolean = !JCharacter.isJavaIdentifierPart(c)
-}
+  def willBeEncoded(c: Char): Boolean = !isJavaIdentifierPart(c)

tests/pos/surrogates.scala

@@ -25,4 +25,8 @@ class Construction {
   def reversed = "xyz\udc00\ud801abc"
 }
 
+class Demon {
+  val 😈 = 42
+}
+
 // was: error: illegal character '\ud801', '\udc00'

tests/pos/t1406.scala

@@ -0,0 +1,34 @@
+
+class Identifiers {
+
+  def f(x: Any): Boolean = x match {
+    case 𐐨XYZ: String => true
+    case 𐐨                => true
+  }
+  def g(x: Any) = x match {
+    case 𐐨 @ _ => 𐐨
+  }
+}
+class Ops {
+  def 𝆗 = 42        // was error: illegal character
+  def op_𝆗 = 42     // was error: illegal character
+  def 🌀 = 42
+  def op_🌀 = 42
+  def 🚀 = 42
+  def op_🚀 = 42
+  def 🜀 = 42
+  def op_🜀 = 42
+  def 𝓅 = 42
+  def op_𝓅 = 42
+}
+class Strings {
+  implicit class Interps(sc: StringContext) {
+    def 𝓅(parts: Any*) = "done"
+  }
+  /*
+  def 𝓅 = 42
+  def interpolated = s"$𝓅"
+  def e = "a 𝓅 b"
+  */
+  def f = 𝓅"one"
+}

tests/run/t1406b.scala

@@ -0,0 +1,23 @@
+
+case class C(n: Int) {
+  def 𐀀(c: C): C = C(n * c.n)   // actually a letter but supplementary 0x10000
+  def ☀(c: C): C = C(n * c.n)   // just a symbol
+  def ☀=(c: C): C = C(n * c.n)   // just a symbol
+  def 🌀(c: C): C = C(n * c.n)  // cyclone operator is symbol, supplementary
+  def 🌀=(c: C): C = C(n * c.n)  // cyclone operator is symbol, supplementary
+  def *(c: C): C = C(n * c.n)
+  def +(c: C): C = C(n + c.n)
+}
+object Test extends App {
+  val c, d = C(42)
+  println(c + d)
+  println(c * d)
+  println(c ☀ d)
+  println(c * d + d)
+  println(c ☀ d + d)
+  println(c ☀= d + d)           // assignment op is low precedence
+  println(c 𐀀 d + d)            // the first one, letter should be low precedence
+  println(c 🌀d + d)            // the second one, cyclone should be high precedence
+  println(c 🌀= d + d)            // the second one, cyclone should be high precedence
+}
+