add tokens datatype

2026-05-05 16:32:25 +01:00
parent e0be1c7a3a
commit 152b7e9740
3 changed files with 72 additions and 5 deletions
--- a/compiler/token.sml
+++ b/compiler/token.sml
@@ -0,0 +1,56 @@
+structure Token =
+struct
+  datatype t =
+    AND
+  | ANDALSO
+  | AS
+  | CASE
+  | DATATYPE
+  | ELSE
+  | END
+  | EXCEPTION
+  | FN
+  | FUN
+  | HANDLE
+  | IF
+  | IN
+  | INFIX
+  | INFIXR
+  | LET
+  | LOCAL
+  | NONFIX
+  | OF
+  | OP
+  | OPEN
+  | ORELSE
+  | RAISE
+  | REC
+  | THEN
+  | TYPE
+  | VAL
+  | WITH
+  | WITHTYPE
+  | WHILE
+  | L_PAREN
+  | R_PAREN
+  | L_SQUARE_BRACKET
+  | R_SQUARE_BRACKET
+  | L_BRACE
+  | R_BRACE
+  | COMMA
+  | COLON
+  | SEMICOLON
+  | TRIPLE_DOT
+  | UNDERSCORE
+  | PIPE
+  | EQ_ARROW
+  | DASH_ARROW
+  | HASH
+
+  | BOOL of bool
+  | INT of int
+  | WORD of word
+  | REAL of real
+  | CHAR of char
+  | STRING of string
+end