begin preparation to remove 'word_type' datatype in cursor.sml, which will simplify implementation of word motions

fcore/cursor.sml

@@ -858,82 +858,172 @@ struct
     else
       NON_BLANK
 
-  fun helpNextWordString 
+  fun isNextChrSpace (strPos, str, strTl) =
-    ( strPos, str, absIdx 
+    if strPos + 1 < String.size str then
-    , strTl, lineTl 
+      let
-    , prevWordType, hasPassedSpace) =
+        val chr = String.sub (str, strPos + 1)
-      if strPos = String.size str then
+      in
-        helpNextWordList 
+        Char.isSpace chr
-          (strTl, lineTl, absIdx, prevWordType, hasPassedSpace)
+      end
-      else
+    else
-        let
+      case strTl of
-          val chr = String.sub (str, strPos)
+        hd :: _ => 
-        in
+          let
-          if Char.isAlphaNum chr orelse chr = #"_" then
+            val chr = String.sub (hd, 0)
-            case prevWordType of
+          in
-              ALPHA_NUM =>
+            Char.isSpace chr
-                (* current chr is ALPHA_NUM
+          end
-                 * and previous chr was also ALPHA_NUM
+      | [] => false
-                 * so continue iterating *)
+
-                helpNextWordString
+  fun notIsNextChrSpace (strPos, str, strTl) =
-                  ( strPos + 1, str, absIdx + 1
+    let
-                  , strTl, lineTl 
+      val isSpace = isNextChrSpace (strPos, str, strTl)
-                  , ALPHA_NUM, hasPassedSpace
+    in
-                  )
+      not isSpace
-            | SPACE =>
+    end
-                (* we moved from SPACE to ALPHA_NUM, 
+
-                 * meaning we may have reached a new word. *)
+  fun isNextChrNonBlank (strPos, str, strTl) =
-                if hasPassedSpace then
+    if strPos + 1 < String.size str then
-                  absIdx
+      let
-                else
+        val chr = String.sub (str, strPos + 1)
-                  helpNextWordString
+        val isNotBlank = 
-                    ( strPos + 1, str, absIdx + 1
+          Char.isSpace chr 
-                    , strTl, lineTl 
+          orelse Char.isAlphaNum chr 
-                    , ALPHA_NUM, true
+          orelse chr = #"_"
-                    )
+      in
-            | NON_BLANK=>
+        not isNotBlank
-                (* moved from NON_BLANK to ALPHA_NUM,
+      end
-                 * meaning we reached new word *)
+    else
-                 absIdx
+      case strTl of
-          else if Char.isSpace chr then
+        hd :: _ =>
+          let
+            val chr = String.sub (hd, 0)
+            val isNotBlank =
+              Char.isSpace chr
+              orelse Char.isAlphaNum chr
+              orelse chr = #"_"
+          in
+            not isNotBlank
+          end
+      | [] => false
+
+  fun isNextChrAlphaNum (strPos, str, stl) =
+    if strPos + 1 < String.size str then
+      let
+        val chr = String.sub (str, strPos + 1)
+      in
+        Char.isAlphaNum chr orelse chr = #"_"
+      end
+    else
+      case stl of
+        hd :: _ =>
+          let
+            val chr = String.sub (str, 0)
+          in
+            Char.isAlphaNum chr orelse chr = #"_"
+          end
+      | [] => false
+
+  fun isPrevChrSpace (strPos, str, strTl) =
+    if strPos > 0 then
+      let
+        val prevChr = String.sub (str, strPos - 1)
+      in
+        Char.isSpace prevChr
+      end
+    else
+      case strTl of
+        hd :: _ =>
+          let
+            val prevChr = String.sub (hd, String.size hd - 1)
+          in
+            Char.isSpace prevChr
+          end
+      | [] => false
+
+  fun notIsPrevChrSpace (strPos, str, strTl) =
+    let
+      val isSpace = isPrevChrSpace (strPos, str, strTl)
+    in
+      not isSpace
+    end
+
+  fun isPrevChrAlphaNum (strPos, str, strTl) =
+    if strPos > 0 then
+      let
+        val chr = String.sub (str, strPos - 1)
+      in
+        Char.isAlphaNum chr orelse chr = #"_"
+      end
+    else
+      case strTl of
+        hd :: _ =>
+          let
+            val chr = String.sub (hd, String.size hd - 1)
+          in
+            Char.isAlphaNum chr orelse chr = #"_"
+          end
+      | [] => false
+
+  fun isPrevChrNonBlank (strPos, str, strTl) =
+    if strPos > 0 then
+      let
+        val chr = String.sub (str, strPos - 1)
+        val isNotBlank = 
+          Char.isSpace chr 
+          orelse Char.isAlphaNum chr 
+          orelse chr = #"_"
+      in
+        not isNotBlank
+      end
+    else
+      case strTl of
+        hd :: _ =>
+          let
+            val chr = String.sub (hd, String.size hd - 1)
+            val isNotBlank = 
+              Char.isSpace chr 
+              orelse Char.isAlphaNum chr 
+              orelse chr = #"_"
+          in
+            not isNotBlank
+          end
+      | [] => false
+
+  fun helpNextWordString (strPos, str, absIdx , strTl, lineTl) =
+    if strPos = String.size str then
+      helpNextWordList (strTl, lineTl, absIdx)
+    else
+      let
+        val chr = String.sub (str, strPos)
+      in
+        if Char.isAlphaNum chr orelse chr = #"_" then
+          if isNextChrNonBlank (strPos, str, strTl) then
+            absIdx + 1
+          else
+            helpNextWordString
+              (strPos + 1, str, absIdx + 1, strTl, lineTl)
+        else if Char.isSpace chr then
+          if notIsNextChrSpace (strPos, str, strTl) then
+            absIdx + 1
+          else
             (* nothing to do on space, except keep iterating *)
             helpNextWordString
-              ( strPos + 1, str, absIdx + 1
+              (strPos + 1, str, absIdx + 1, strTl, lineTl)
-              , strTl, lineTl 
+        else
-              , SPACE, true
+          (* chr is NON_BLANK. *)
-              )
+          if isNextChrAlphaNum (strPos, str, strTl) then
+            absIdx + 1
           else
-            (* chr is NON_BLANK. *)
+            helpNextWordString
-            case prevWordType of
+              (strPos + 1, str, absIdx + 1, strTl, lineTl)
-              NON_BLANK =>
+      end
-                (* moved from non-blank to non-blank
-                 * so keep iterating *)
-                helpNextWordString
-                  ( strPos + 1, str, absIdx + 1
-                  , strTl, lineTl
-                  , NON_BLANK, hasPassedSpace
-                  )
-            | ALPHA_NUM => 
-                (* moved from ALPHA_NUM to non-blank
-                 * so we have reached new word *)
-                absIdx
-            | SPACE =>
-                (* from space to non-blank *)
-                if hasPassedSpace then
-                  absIdx
-                else
-                  helpNextWordString
-                    ( strPos + 1, str, absIdx + 1
-                    , strTl, lineTl
-                    , NON_BLANK, true
-                    )
-        end
 
-  and helpNextWordList (strings, lines, absIdx, prevWordType, hasPassedSpace) =
+  and helpNextWordList (strings, lines, absIdx) =
     case (strings, lines) of
       (strHd::strTl, lineHd::lineTl) =>
         helpNextWordString
-          ( 0, strHd, absIdx, strTl, lineTl, prevWordType, hasPassedSpace )
+          (0, strHd, absIdx, strTl, lineTl)
     | (_, _) =>
         (* reached end of lineGap; 
          * return last valid chr position *)
@@ -942,11 +1032,9 @@ struct
   fun startNextWord (shd, strIdx, absIdx, stl, ltl) =
     let
       val chr = String.sub (shd, strIdx)
-      val wordType = getWordType chr
-      val isSpace = wordType = SPACE
     in
       helpNextWordString 
-        (strIdx + 1, shd, absIdx + 1, stl, ltl, wordType, isSpace)
+        (strIdx, shd, absIdx, stl, ltl)
     end
 
   fun nextWord (lineGap: LineGap.t, cursorIdx) =
@@ -977,23 +1065,6 @@ struct
         | (_, _) => cursorIdx
     end
 
-  fun isPrevChrSpace (strPos, str, strTl) =
-    if strPos > 0 then
-      let
-        val prevChr = String.sub (str, strPos - 1)
-      in
-        Char.isSpace prevChr
-      end
-    else
-      case strTl of
-        hd :: _ =>
-          let
-            val prevChr = String.sub (hd, String.size hd - 1)
-          in
-            Char.isSpace prevChr
-          end
-      | [] => true
-
   fun helpPrevWordString 
     (strPos, str, absIdx, strTl, lineTl, lastWordType) =
       if strPos < 0 then
@@ -1142,65 +1213,6 @@ struct
       | (_, _) => cursorIdx
     end
 
-  fun isNextChrSpace (strPos, str, strTl) =
-    if strPos - 1 < String.size str then
-      let
-        val chr = String.sub (str, strPos + 1)
-      in
-        Char.isSpace chr
-      end
-    else
-      case strTl of
-        hd :: _ => 
-          let
-            val chr = String.sub (hd, 0)
-          in
-            Char.isSpace chr
-          end
-      | [] => false
-
-  fun isNextChrNonBlank (strPos, str, strTl) =
-    if strPos - 1 < String.size str then
-      let
-        val chr = String.sub (str, strPos + 1)
-        val isNotBlank = 
-          Char.isSpace chr 
-          orelse Char.isAlphaNum chr 
-          orelse chr = #"_"
-      in
-        not isNotBlank
-      end
-    else
-      case strTl of
-        hd :: _ =>
-          let
-            val chr = String.sub (hd, 0)
-            val isNotBlank =
-              Char.isSpace chr
-              orelse Char.isAlphaNum chr
-              orelse chr = #"_"
-          in
-            not isNotBlank
-          end
-      | [] => false
-
-  fun isNextChrAlphaNum (strPos, str, stl) =
-    if strPos - 1 < String.size str then
-      let
-        val chr = String.sub (str, strPos + 1)
-      in
-        Char.isAlphaNum chr orelse chr = #"_"
-      end
-    else
-      case stl of
-        hd :: _ =>
-          let
-            val chr = String.sub (str, strPos + 1)
-          in
-            Char.isAlphaNum chr orelse chr = #"_"
-          end
-      | [] => false
-
   fun helpEndOfWordString 
     (strPos, str, absIdx, stl, ltl) =
       if strPos = String.size str then