add tokens datatype

compiler/scanner.sml

@@ -7,10 +7,8 @@ struct
   val realDfa = Dfa.fromString "[0-9]+.[0-9]+"
 
   val idDfa = Dfa.fromString "[a-zA-Z][a-zA-Z0-9_']*"
-  val longIdDfa =
-    Dfa.fromString "[a-zA-Z[(a-zA-Z0-9_']*(\\.[a-zA-Z][a-zA-Z0-9_']*)+"
-
-      Hello.world
+  val longIdDfa = Dfa.fromString
+    "[a-zA-Z[(a-zA-Z0-9_']*(\\.[a-zA-Z][a-zA-Z0-9_']*)+"
 
   fun skipFormattingChars (str, pos) =
     if pos = String.size str then
@@ -60,4 +58,16 @@ struct
       | chr => helpExtractString (pos + 1, str, acc ^ String.implode [chr])
 
   fun extractString (pos, str) = helpExtractString (pos, str, "")
+
+  fun extractChar (pos, str) =
+    let
+      val (nextPos, newString) = extractString (pos, str)
+    in
+      if String.size newString = 1 then
+        (nextPos, newString)
+      else
+        raise Fail
+          ("expected char of size 1 but got char of size"
+           ^ Int.toString (String.size newString) ^ "\n")
+    end
 end