add tokens datatype

compiler/scanner.sml

@@ -7,10 +7,8 @@ struct
   val realDfa = Dfa.fromString "[0-9]+.[0-9]+"
 
   val idDfa = Dfa.fromString "[a-zA-Z][a-zA-Z0-9_']*"
-  val longIdDfa =
-    Dfa.fromString "[a-zA-Z[(a-zA-Z0-9_']*(\\.[a-zA-Z][a-zA-Z0-9_']*)+"
-
-      Hello.world
+  val longIdDfa = Dfa.fromString
+    "[a-zA-Z[(a-zA-Z0-9_']*(\\.[a-zA-Z][a-zA-Z0-9_']*)+"
 
   fun skipFormattingChars (str, pos) =
     if pos = String.size str then
@@ -60,4 +58,16 @@ struct
       | chr => helpExtractString (pos + 1, str, acc ^ String.implode [chr])
 
   fun extractString (pos, str) = helpExtractString (pos, str, "")
+
+  fun extractChar (pos, str) =
+    let
+      val (nextPos, newString) = extractString (pos, str)
+    in
+      if String.size newString = 1 then
+        (nextPos, newString)
+      else
+        raise Fail
+          ("expected char of size 1 but got char of size"
+           ^ Int.toString (String.size newString) ^ "\n")
+    end
 end

compiler/token.sml

@@ -0,0 +1,56 @@
+structure Token =
+struct
+  datatype t =
+    AND
+  | ANDALSO
+  | AS
+  | CASE
+  | DATATYPE
+  | ELSE
+  | END
+  | EXCEPTION
+  | FN
+  | FUN
+  | HANDLE
+  | IF
+  | IN
+  | INFIX
+  | INFIXR
+  | LET
+  | LOCAL
+  | NONFIX
+  | OF
+  | OP
+  | OPEN
+  | ORELSE
+  | RAISE
+  | REC
+  | THEN
+  | TYPE
+  | VAL
+  | WITH
+  | WITHTYPE
+  | WHILE
+  | L_PAREN
+  | R_PAREN
+  | L_SQUARE_BRACKET
+  | R_SQUARE_BRACKET
+  | L_BRACE
+  | R_BRACE
+  | COMMA
+  | COLON
+  | SEMICOLON
+  | TRIPLE_DOT
+  | UNDERSCORE
+  | PIPE
+  | EQ_ARROW
+  | DASH_ARROW
+  | HASH
+
+  | BOOL of bool
+  | INT of int
+  | WORD of word
+  | REAL of real
+  | CHAR of char
+  | STRING of string
+end

compiler/uses.sml

@@ -1,2 +1,3 @@
 use "../dfa-gen/dfa-gen.sml";
-use "scanner.sml"
+use "token.sml";
+use "scanner.sml";