implemented include

5 files changed, 404 insertions, 328 deletions

diff --git a/compiler/elymas.ey b/compiler/elymas.ey
index 11a4d33..8e778b1 100755
--- a/compiler/elymas.ey
+++ b/compiler/elymas.ey
@@ -2,12 +2,14 @@
 
 "standard.ey" include
 "elymasGlobal.ey" include
+"elymasTokenize.ey" include
 "elymasLexer.ey" include
 
 sys .argv len not { "Usage: ./elymas <input file.ey>" die } rep
 
 [
   "standard.ey"
+  "elymasTokenize.ey"
   "standardClient.ey"
   0 sys .argv *
 ] {
diff --git a/compiler/elymasGlobalSys.ey b/compiler/elymasGlobalSys.ey
index 9d20c35..01ca564 100644
--- a/compiler/elymasGlobalSys.ey
+++ b/compiler/elymasGlobalSys.ey
@@ -1,7 +1,24 @@
 <
   "sys" enterSubScope
 
+  <
+    # handle an identifier in the current scope according to current quote level
+    # 0 -> identifier to handle
+    [[
+      8 /r15 :subqImm8Reg
+      /r15 :popqMem
+
+      internalExecuteIdentifier /rax :movqImmReg
+      /rax :callqReg
+
+      /r15 :pushqMem
+      8 /r15 :addqImm8Reg
+      :retn
+    ]] /eyexecuteIdentifier defv
+  > _ ==globalFunctions { defv }' ::allocateOffsetStruct
+
   [
+    globalFunctions keys eydeff { | }' createScopeEntries
     createScopeExtensionEntries
   ] :execute
 
diff --git a/compiler/elymasLexer.ey b/compiler/elymasLexer.ey
index a140769..77ae81b 100644
--- a/compiler/elymasLexer.ey
+++ b/compiler/elymasLexer.ey
@@ -1,21 +1,10 @@
 <
-  [ /0 /1 /2 /3 /4 /5 /6 /7 /8 /9 ] ==digits
-
-  { 0 ==result
-    { "(.)(.*)" regex } {
-      { streq }_ digits -01 index result 10 mul add =result
-    } loop
-    result
-  }
-> -- /base10decode deff
-
-<
   { assembler -01 . } ":" deff
   { assemblerLibrary -01 . } "::" deff
   assembler .|label "@" deff
   "%" _ : -01 deff
 
-  { .value base10decode ==v
+  { .value elymas .base10decode ==v
     [
       # allocate int
       16 /rdi :movqImmReg
@@ -44,43 +33,12 @@
   } /TOKID
 > -- 3 |defv rep
 
-{ /f deff -101 /s defv regex { f } { s } ? * } /rxparse deff
-
-{ " " cat
-  { < /handle deff /value defv > } /token deff
-  [ -01 { _ "" streq not } {
-    0 /matched defv { /f deff matched { -- } { { 1 =matched f } rxparse } ? * } /parse deff
-
-    "^ (.*)" { } parse
-    "^#" { "" } parse
-    "^(\\d+) +(.*)" { TOKINT token -01 } parse
-    "^\"(.*)" {
-      "" /str defv
-      { _ "^\"(.*)" regex { -01 -- 0 } { 1 } ? * } {
-        0 /strmatched defv { /f deff strmatched { -- } { { 1 =strmatched f } rxparse } ? * } /strparse deff
-
-        "^\\\\\\\\(.*)" { str "\\" cat =str } strparse
-        "^\\\\n(.*)" { str "\n" cat =str } strparse
-        "^\\\\0(.*)" { str "\0" cat =str } strparse
-        "^\\\\\"(.*)" { str "\"" cat =str } strparse
-        "^([^\"\\\\])(.*)" { str -01 cat =str } strparse
-        strmatched not { "Tokenization of string-like failed" die } rep
-      } loop
-      str TOKSTR token -01
-    } parse
-    "^([^a-zA-Z0-9 ]+)([a-zA-Z0-9][^ ]*) +(.*)" { -201 TOKSTR token " " -1203 cat cat } parse
-    "^([a-zA-Z0-9]+|[^a-zA-Z0-9 ]+) +(.*)" { TOKID token -01 } parse
-
-    matched not { "Tokenization failed: " -01 cat die } rep
-  } loop -- ]
-} /tokenize deff
-
 { /input defv
   "" {
     4096 input .readstr cat
     _ "" streq not
   } {
-    { _ "([^\\n]*)\\n(.*)" regex } { -102 -- tokenize {
+    { _ "([^\\n]*)\\n(.*)" regex } { -102 -- TOKINT TOKSTR TOKID elymas .tokenize {
       _ .handle
       # assemblerLibrary .stackDump
       # assemblerLibrary .globalScopeDump
diff --git a/compiler/elymasTokenize.ey b/compiler/elymasTokenize.ey
new file mode 100644
index 0000000..347a81b
--- /dev/null
+++ b/compiler/elymasTokenize.ey
@@ -0,0 +1,46 @@
+<
+  <
+    [ /0 /1 /2 /3 /4 /5 /6 /7 /8 /9 ] ==digits
+
+    { 0 ==result
+      { "(.)(.*)" regex } {
+        { streq }_ digits -01 index result 10 mul add =result
+      } loop
+      result
+    }
+  > -- /base10decode deff
+
+  { /f deff -101 /s defv regex { f } { s } ? * } /rxparse deff
+
+  { ==TOKID ==TOKSTR ==TOKINT
+    " " cat
+    { < /handle deff /value defv > } /token deff
+    [ -01 { _ "" streq not } {
+      0 /matched defv { /f deff matched { -- } { { 1 =matched f } rxparse } ? * } /parse deff
+
+      "^ (.*)" { } parse
+      "^#" { "" } parse
+      "^(\\d+) +(.*)" { TOKINT token -01 } parse
+      "^\"(.*)" {
+        "" /str defv
+        { _ "^\"(.*)" regex { -01 -- 0 } { 1 } ? * } {
+          0 /strmatched defv { /f deff strmatched { -- } { { 1 =strmatched f } rxparse } ? * } /strparse deff
+
+          "^\\\\\\\\(.*)" { str "\\" cat =str } strparse
+          "^\\\\n(.*)" { str "\n" cat =str } strparse
+          "^\\\\0(.*)" { str "\0" cat =str } strparse
+          "^\\\\\"(.*)" { str "\"" cat =str } strparse
+          "^([^\"\\\\])(.*)" { str -01 cat =str } strparse
+          strmatched not { "Tokenization of string-like failed" die } rep
+        } loop
+        str TOKSTR token -01
+      } parse
+      "^([^a-zA-Z0-9 ]+)([a-zA-Z0-9][^ ]*) +(.*)" { -201 TOKSTR token " " -1203 cat cat } parse
+      "^([a-zA-Z0-9]+|[^a-zA-Z0-9 ]+) +(.*)" { TOKID token -01 } parse
+
+      matched not { "Tokenization failed: " -01 cat die } rep
+    } loop -- ]
+  } /tokenize deff
+> /elymas defv
+
+# vim: syn=elymas
diff --git a/compiler/standardClient.ey b/compiler/standardClient.ey
index 01d04a5..89f777e 100644
--- a/compiler/standardClient.ey
+++ b/compiler/standardClient.ey
@@ -432,290 +432,313 @@
 > -- /dump deff
 
 # # regex support
-# # ideas taken from http://swtch.com/~rsc/regexp/regexp3.html
-# {
-#   0 ==MATCH 1 ==TERM 2 ==JUMP 3 ==SPLIT 4 ==SAVE 5 ==FIRST 6 ==LAST
-# 
-#   { ==b ==a [
-#     [ SPLIT 1 a len 1 add ] # FIXME this should be "2 add"?!
-#     a _ len dearray
-#     [ JUMP b len ]
-#     b _ len dearray
-#   ] } /alternative deff
-# 
-#   |cat /sequence deff
-# 
-#   { ==a [ # TODO measure separate + implementation performance impact
-#     [ JUMP a len 1 add ]
-#     a _ len dearray
-#     [ SPLIT 1 a len neg ]
-#   ] } /star deff
-# 
-#   { ==p [
-#     [ TERM p ]
-#   ] } /terminal deff
-# 
-#   { ==i ==a [
-#     [ SAVE i 2 mul ]
-#     a _ len dearray
-#     [ SAVE i 2 mul 1 add ]
-#   ] } /capture deff
-# 
-#   { [ ] } /empty deff
-# 
-#   { ==str
-#     str len 0 eq {
-#       1 neg
-#     } {
-#       0 str *
-#     } ? *
-#   } /head deff
-# 
-#   { 1 -01 str .postfix } /tail deff
-# 
-#   { 0 -01 * -101 head eq } "^" deff
-#   { deff }' /install deff
-#   [ "(" ")" "[" "]" "-" "|" "^" "*" "+" "." "$" "\\" ] { ==c
-#     { _ head 0 c * eq } "^" c cat install
-#   } each
-# 
-#   { # "(parse) re: " -101 cat dump
-# 
-#     seq ==a
-#     ^| {
-#       tail parse ==b
-#       a b alternative =a
-#     } rep
-#     a
-#   } /parse deff
-# 
-#   { # "(seq) re: " -101 cat dump
-# 
-#     empty _ ==a
-#             ==l
-# 
-#     { # "(seq loop) re: " -101 cat dump
-#       _ head 1 neg eq -01
-#       ^| -01
-#       ^) -01
-#       -0321 or or not
-#     } {
-#       { ^* } {
-#         l star =l
-#         tail
-#       } { ^+ } {
-#         l l star sequence =l
-#         tail
-#       } {
-#         a l sequence =a
-#         atom =l
-#       } ifthenelse ifthenelse *
-#     } loop
-#     a l sequence
-#   } /seq deff
-# 
-#   { ==e ==t =*i
-#     { i t e ? * }
-#   } /ifthenelse deff
-# 
-#   0 ==currentCapture
-#   
-#   { # "(atom) re: " -101 cat dump
-#     empty ==a
-# 
-#     { ^( } {
-#       tail parse currentCapture capture =a
-#       currentCapture 1 add =currentCapture
-#       ^) not { ") expected" die } rep
-#       tail
-#     } { ^[ } {
-#       tail
-#       ^^ {
-#         tail chars =*nset
-#         { nset not } ==set
-#         ^] not { "] expected" die } rep
-#         tail
-#       }' {
-#         chars ==set
-#         ^] not { "] expected" die } rep
-#         tail
-#       }' ? *
-#       set terminal =a
-#     } { ^. } {
-#       { -- 1 } terminal =a
-#       tail
-#     } { ^^ } {
-#       [ [ FIRST ] ] =a
-#       tail
-#     } { ^$ } {
-#       [ [ LAST ] ] =a
-#       tail
-#     } { ^\ } {
-#       tail
-#       { ^d } {
-#         { _ 0 "0" * ge -01 0 "9" * le and } terminal =a
-#         tail
-#       } { ^\ } {
-#         { 0 "\\" * eq } terminal =a
-#         tail
-#       } {
-#         "invalid character after \\ in regex" die
-#       } ifthenelse ifthenelse *
-#     } {
-#       _ head { eq }_ terminal =a
-#       tail
-#     } ifthenelse ifthenelse ifthenelse ifthenelse ifthenelse ifthenelse *
-# 
-#     # "(atom end) re: " -101 cat dump
-#     a
-#   } /atom deff
-# 
-#   { # "(chars) re: " -101 cat dump
-#     ^] {
-#       tail chars2 =*s
-#       { _ s -01 0 "]" * eq or } ==set
-#     }' {
-#       chars2 ==set
-#     }' ? *
-#     set
-#   } /chars deff
-# 
-#   { # "(chars2) re: " -101 cat dump
-#     ^- {
-#       tail chars2 =*s
-#       { _ s -01 0 "-" * eq or } ==set
-#     }' {
-#       charsR ==set
-#     }' ? *
-#     set
-#   } /chars2 deff
-# 
-#   { # "(charsR) re: " -101 cat dump
-#     charsN ==set
-#     { ^] not } {
-#       set =*s1
-#       charsN =*s2
-#       { _ s1 -01 s2 or } =set
-#     } loop
-#     set
-#   } /charsR deff
-# 
-#   { # "(charsN) re: " -101 cat dump
-#     _ head ==start
-#     tail
-#     ^- {
-#       tail
-#       _ head ==end
-#       tail
-#       { _ start ge -01 end le and } ==set
-#     }' {
-#       { start eq } ==set
-#     }' ? *
-#     set
-#   } /charsN deff
-# 
-#   { <
-#     0 ==pc
-#     [ 10 { 0 0 } rep ] ==captures
-#   > } /newThread deff
-# 
-#   |add /origadd deff
-# 
-#   # TODO think about implementation efficiency
-#   { < ==maxSize
-#     0 ==size
-#     [ maxSize { 0 } rep ] =*get
-#     [ maxSize { 1 } rep ] =*pcFree
-# 
-#     { ==thread
-#       thread .pc pcFree {
-#         thread size |get =[]
-#         0 thread .pc |pcFree =[]
-#         size 1 origadd =size
-#       } rep
-#     } /add deff
-# 
-#     {
-#       0 =size
-#       [ maxSize { 1 } rep ] =pcFree
-#     } /clear deff
-#   > } /threadList deff
-# 
-#   { ==thread ==newpc <
-#     newpc ==pc
-#     [ thread .captures 20 dearray ] ==captures
-#   > } /cloneThread deff
-# 
-#   { ==prog ==string
-#     0 ==position
-#     string len ==maxPosition
-#     0 ==done
-#     0 ==matched
-# 
-#     prog len _ threadList ==clist
-#                threadList ==nlist
-# 
-#     newThread _ ==thread clist .add
-# 
-#     { position maxPosition le done not and } {
-#       0 ==i
-#       { i clist .size lt done not and } {
-#         i clist .get _ =thread
-#                        .pc _ ==pc
-#                              prog * =*code
-#         0 code [
-#           { # MATCH
-#             1 =matched
-#             clist .clear
-#           } { # TERM
-#             position maxPosition lt {
-#               position string * 1 code * { pc 1 add thread cloneThread nlist .add } rep
-#             } rep
-#           } { # JUMP
-#             pc 1 code add thread cloneThread clist .add
-#           } { # SPLIT
-#             pc 1 code add thread cloneThread clist .add
-#             pc 2 code add thread cloneThread clist .add
-#           } { # SAVE
-#             position 1 code thread .captures =[]
-#             pc 1 add thread cloneThread clist .add
-#           } { # FIRST
-#             position 0 eq { pc 1 add thread cloneThread clist .add } rep
-#           } { # LAST
-#             position maxPosition eq { pc 1 add thread cloneThread clist .add } rep
-#           }
-#         ] * *
-#         i 1 add =i
-#       } loop
-# 
-#       clist nlist =clist =nlist
-#       nlist .clear
-#       position 1 add =position
-#     } loop
-# 
-#     matched {
-#       currentCapture ==i
-#       { i } { i 1 sub =i
-#         string
-#         i 2 mul       thread .captures * _ ==start -01 str .postfix
-#         i 2 mul 1 add thread .captures * start sub -01 str .inplacePrefix
-#       } loop
-#     } rep
-#     matched
-#   } /execute deff
-# 
-#   parse ==prog --
-#   [
-#     [ SPLIT 3 1 ]
-#     [ TERM { -- 1 } ]
-#     [ JUMP 2 neg ]
-#     prog _ len dearray
-#     [ MATCH ]
-#   ] =prog
-#   { prog execute }
-# } /enregex deff
-# 
-# { enregex * } /regex deff
+# ideas taken from http://swtch.com/~rsc/regexp/regexp3.html
+{
+  0 ==MATCH 1 ==TERM 2 ==JUMP 3 ==SPLIT 4 ==SAVE 5 ==FIRST 6 ==LAST
+
+  { ==b ==a [
+    [ SPLIT 1 a len 1 add ] # FIXME this should be "2 add"?!
+    a _ len dearray
+    [ JUMP b len ]
+    b _ len dearray
+  ] } /alternative deff
+
+  |cat /sequence deff
+
+  { ==a [ # TODO measure separate + implementation performance impact
+    [ JUMP a len 1 add ]
+    a _ len dearray
+    [ SPLIT 1 a len neg ]
+  ] } /star deff
+
+  { ==p [
+    [ TERM p ]
+  ] } /terminal deff
+
+  { ==i ==a [
+    [ SAVE i 2 mul ]
+    a _ len dearray
+    [ SAVE i 2 mul 1 add ]
+  ] } /capture deff
+
+  { [ ] } /empty deff
+
+  { ==str
+    str len 0 eq {
+      1 neg
+    } {
+      0 str *
+    } ? *
+  } /head deff
+
+  { 1 -01 str .postfix } /tail deff
+
+  { 0 -01 * -101 head eq } "^" deff
+  { deff }' /install deff
+  [ "(" ")" "[" "]" "-" "|" "^" "*" "+" "." "$" "\\" ] { ==c
+    { _ head 0 c * eq } "^" c cat install
+  } each
+
+  { # "(parse) re: " -101 cat dump
+
+    seq ==a
+    ^| {
+      tail parse ==b
+      a b alternative =a
+    } rep
+    a
+  } /parse deff
+
+  { # "(seq) re: " -101 cat dump
+
+    empty _ ==a
+            ==l
+
+    { # "(seq loop) re: " -101 cat dump
+      _ head 1 neg eq -01
+      ^| -01
+      ^) -01
+      -0321 or or not
+    } {
+      { ^* } {
+        l star =l
+        tail
+      } { ^+ } {
+        l l star sequence =l
+        tail
+      } {
+        a l sequence =a
+        atom =l
+      } ifthenelse ifthenelse *
+    } loop
+    a l sequence
+  } /seq deff
+
+  { ==e ==t =*i
+    { i t e ? * }
+  } /ifthenelse deff
+
+  0 ==currentCapture
+  
+  { # "(atom) re: " -101 cat dump
+    empty ==a
+
+    { ^( } {
+      tail parse currentCapture capture =a
+      currentCapture 1 add =currentCapture
+      ^) not { ") expected" die } rep
+      tail
+    } { ^[ } {
+      tail
+      ^^ {
+        tail chars =*nset
+        { nset not } ==set
+        ^] not { "] expected" die } rep
+        tail
+      }' {
+        chars ==set
+        ^] not { "] expected" die } rep
+        tail
+      }' ? *
+      set terminal =a
+    } { ^. } {
+      { -- 1 } terminal =a
+      tail
+    } { ^^ } {
+      [ [ FIRST ] ] =a
+      tail
+    } { ^$ } {
+      [ [ LAST ] ] =a
+      tail
+    } { ^\ } {
+      tail
+      { ^d } {
+        { _ 0 "0" * ge -01 0 "9" * le and } terminal =a
+        tail
+      } { ^\ } {
+        { 0 "\\" * eq } terminal =a
+        tail
+      } { ^n } {
+        { 0 "\n" * eq } terminal =a
+        tail
+      } {
+        "invalid character '" "' after \\ in regex" -120 cat cat die
+      } ifthenelse ifthenelse ifthenelse *
+    } {
+      _ head { eq }_ terminal =a
+      tail
+    } ifthenelse ifthenelse ifthenelse ifthenelse ifthenelse ifthenelse *
+
+    # "(atom end) re: " -101 cat dump
+    a
+  } /atom deff
+
+  { # "(chars) re: " -101 cat dump
+    ^] {
+      tail chars2 =*s
+      { _ s -01 0 "]" * eq or } ==set
+    }' {
+      chars2 ==set
+    }' ? *
+    set
+  } /chars deff
+
+  { # "(chars2) re: " -101 cat dump
+    ^- {
+      tail chars2 =*s
+      { _ s -01 0 "-" * eq or } ==set
+    }' {
+      charsR ==set
+    }' ? *
+    set
+  } /chars2 deff
+
+  { # "(charsR) re: " -101 cat dump
+    charsN ==set
+    { ^] not } {
+      set =*s1
+      charsN =*s2
+      { _ s1 -01 s2 or } =set
+    } loop
+    set
+  } /charsR deff
+
+  { # "(charsN) re: " -101 cat dump
+    _ head ==start
+    ^\ {
+      tail
+      { ^\ } {
+        0 "\\" * =start
+      } { ^n } {
+        0 "\n" * =start
+      } {
+        "invalid character '" "' after \\ in regex" -120 cat cat die
+      } ifthenelse ifthenelse *
+    } rep
+    tail
+    ^- {
+      tail
+      _ head ==end
+      ^\ {
+        tail
+        { ^\ } {
+          0 "\\" * =end
+        } { ^n } {
+          0 "\n" * =end
+        } {
+          "invalid character '" "' after \\ in regex" -120 cat cat die
+        } ifthenelse ifthenelse *
+      } rep
+      tail
+      { _ start ge -01 end le and } ==set
+    }' {
+      { start eq } ==set
+    }' ? *
+    set
+  } /charsN deff
+
+  { <
+    0 ==pc
+    [ 10 { 0 0 } rep ] ==captures
+  > } /newThread deff
+
+  |add /origadd deff
+
+  # TODO think about implementation efficiency
+  { < ==maxSize
+    0 ==size
+    [ maxSize { 0 } rep ] =*get
+    [ maxSize { 1 } rep ] =*pcFree
+
+    { ==thread
+      thread .pc pcFree {
+        thread size |get =[]
+        0 thread .pc |pcFree =[]
+        size 1 origadd =size
+      } rep
+    } /add deff
+
+    {
+      0 =size
+      [ maxSize { 1 } rep ] =pcFree
+    } /clear deff
+  > } /threadList deff
+
+  { ==thread ==newpc <
+    newpc ==pc
+    [ thread .captures 20 dearray ] ==captures
+  > } /cloneThread deff
+
+  { ==prog ==string
+    0 ==position
+    string len ==maxPosition
+    0 ==done
+    0 ==matched
+
+    prog len _ threadList ==clist
+               threadList ==nlist
+
+    newThread _ ==thread clist .add
+
+    { position maxPosition le done not and } {
+      0 ==i
+      { i clist .size lt done not and } {
+        i clist .get _ =thread
+                       .pc _ ==pc
+                             prog * =*code
+        0 code [
+          { # MATCH
+            1 =matched
+            clist .clear
+          } { # TERM
+            position maxPosition lt {
+              position string * 1 code * { pc 1 add thread cloneThread nlist .add } rep
+            } rep
+          } { # JUMP
+            pc 1 code add thread cloneThread clist .add
+          } { # SPLIT
+            pc 1 code add thread cloneThread clist .add
+            pc 2 code add thread cloneThread clist .add
+          } { # SAVE
+            position 1 code thread .captures =[]
+            pc 1 add thread cloneThread clist .add
+          } { # FIRST
+            position 0 eq { pc 1 add thread cloneThread clist .add } rep
+          } { # LAST
+            position maxPosition eq { pc 1 add thread cloneThread clist .add } rep
+          }
+        ] * *
+        i 1 add =i
+      } loop
+
+      clist nlist =clist =nlist
+      nlist .clear
+      position 1 add =position
+    } loop
+
+    matched {
+      currentCapture ==i
+      { i } { i 1 sub =i
+        string
+        i 2 mul       thread .captures * _ ==start -01 str .postfix
+        i 2 mul 1 add thread .captures * start sub -01 str .inplacePrefix
+      } loop
+    } rep
+    matched
+  } /execute deff
+
+  parse ==prog --
+  [
+    [ SPLIT 3 1 ]
+    [ TERM { -- 1 } ]
+    [ JUMP 2 neg ]
+    prog _ len dearray
+    [ MATCH ]
+  ] =prog
+  { prog execute }
+} /enregex deff
+
+{ enregex * } /regex deff
 
 { ==filename # ==f (left on the stack and executed from sys .asm .programStart)
   sys .asm .patchProgramStart
@@ -916,4 +939,34 @@
   out .close
 } /freeze sys .deff
 
+{ .value sys .executeIdentifier }'
+<
+  /TOKID defv # weird scoping so executeIdentifier is executed in global scope
+  { .value elymas .base10decode } /TOKINT defv
+  { .value } /TOKSTR defv
+
+  # no long-term stack use here as the executed program uses it as well
+  { /input defv
+    "" ==buffer
+    {
+      buffer 4096 input .read cat =buffer # FIXME interpreter API should also have .read defined as returning string
+      buffer "" streq not
+    } {
+      {
+        buffer "([^\\n]*)\\n(.*)" regex
+      } { ==line =buffer
+        line TOKINT TOKSTR TOKID elymas .tokenize {
+          _ .handle
+        } each
+      } loop
+    } loop
+  } /executeFile deff
+
+  { # ==filename
+    sys .file -0010 .open
+                    executeFile
+                    .close
+  }
+> -- /include deff
+
 # vim: syn=elymas