Merge pull request #3082 from Shopify/non-ascii-identifiers

Support non-ASCII identifiers

Author: soutaro

.gitattributes

@@ -0,0 +1 @@
+src/lexer.c linguist-generated

docs/syntax.md

@@ -48,6 +48,18 @@ _proc_ ::= `^` _parameters?_ _self-type-binding?_ _block?_ `->` _type_
          | `^` `(` `?` `)` `->` _type_                                   # Proc type with untyped parameter
 ```
 
+`\w` above, and everywhere else in this document, is `[a-zA-Z0-9_]` together
+with every character outside ASCII -- the same set Ruby takes into an
+identifier. So `ServicioÚltimaVez` is a class name and `nombre_único` is an
+alias name.
+
+The leading character is the exception. RBS reads it to tell a class name from
+an interface name from an alias name, so where it makes that distinction it has
+to be ASCII: `class 日本語` is a class in Ruby but not a name RBS can write.
+Nowhere else is restricted -- a method name, a variable name, an instance
+variable name and a class variable name may all open with any character Ruby
+accepts.
+
 ### Class instance type
 
 Class instance type denotes _an instance of a class_.

include/rbs/lexer.h

@@ -72,18 +72,19 @@ enum RBSTokenType {
     kRETURN,       /* return */
     kMODULESELF,   /* module-self */
 
-    tLIDENT,    /* Identifiers starting with lower case */
-    tUIDENT,    /* Identifiers starting with upper case */
-    tULIDENT,   /* Identifiers starting with `_` followed by upper case */
-    tULLIDENT,  /* Identifiers starting with `_` followed by lower case */
-    tGIDENT,    /* Identifiers starting with `$` */
-    tAIDENT,    /* Identifiers starting with `@` */
-    tA2IDENT,   /* Identifiers starting with `@@` */
-    tBANGIDENT, /* Identifiers ending with `!` */
-    tEQIDENT,   /* Identifiers ending with `=` */
-    tQIDENT,    /* Quoted identifier */
-    pAREF_OPR,  /* [] */
-    tOPERATOR,  /* Operator identifier */
+    tLIDENT,        /* Identifiers starting with lower case */
+    tUIDENT,        /* Identifiers starting with upper case */
+    tULIDENT,       /* Identifiers starting with `_` followed by upper case */
+    tULLIDENT,      /* Identifiers starting with `_` followed by lower case */
+    tNONASCIIIDENT, /* Identifiers starting with a character outside ASCII */
+    tGIDENT,        /* Identifiers starting with `$` */
+    tAIDENT,        /* Identifiers starting with `@` */
+    tA2IDENT,       /* Identifiers starting with `@@` */
+    tBANGIDENT,     /* Identifiers ending with `!` */
+    tEQIDENT,       /* Identifiers ending with `=` */
+    tQIDENT,        /* Quoted identifier */
+    pAREF_OPR,      /* [] */
+    tOPERATOR,      /* Operator identifier */
 
     tCOMMENT,       /* Comment */
     tLINECOMMENT,   /* Comment of all line */