Use separate GC stack to prevent overflow

1 files changed, 51 insertions, 1 deletions

diff --git a/compiler/elymasAsmLib.ey b/compiler/elymasAsmLib.ey
index b830a48..72d0a71 100644
--- a/compiler/elymasAsmLib.ey
+++ b/compiler/elymasAsmLib.ey
@@ -17,6 +17,10 @@
       { sys .?linux }   { 32 }
       { sys .?freebsd } { 4096 }
     ] conds ==MAP_ANONYMOUS
+
+    sys .?linux {
+      256 ==MAP_GROWSDOWN
+    }' rep
   > ==MMAP
 
   <
@@ -102,6 +106,8 @@
   { :labelRecord [ } "[[" deff
   { ] :labelResolve stringResolve } "]]" deff
 
+  { %8F %FF %FF %FF %FF %78 %00 %00 } /GCSTACKSTART deff
+  { %00 %F0 %FF %FF %FF %78 %00 %00 } /GCSTACKMAP deff
   { %00 %00 %00 %00 %00 %60 %00 %00 } /HEAPBASE deff
   { %00 %00 %00 %00 %00 %50 %00 %00 } /BLOCKBASE deff
   { %00 %00 %00 %00 %00 %40 %00 %00 } /MARKBASE deff
@@ -128,6 +134,9 @@
     # amortized mark and sweep costs
     [ %00 %00 %00 %00 %00 %00 %00 %00 ] ==markAndSweepCosts
 
+    # separate GC runtime stack, or zero
+    [ %00 %00 %00 %00 %00 %00 %00 %00 ] ==garbageCollectStack
+
     # freelists for various sizes
     [ FREELISTCOUNT { %00 %00 %00 %00 %00 %00 %00 %00 } rep ] ==freeLists
   > { defv }' allocateOffsetStruct
@@ -670,6 +679,34 @@
       /rax /rax :xorqRegReg
       :retn
 
+      # allocates separate GC stack, stack start returned in rax
+      @allocateGCStack
+      [
+        { sys .?linux } {
+          [ /rcx /r8 /r9 /r10 /r11 /r12 /r13 /r14 /r15 ] { :pushqReg } each
+          SYSCALL .mmap /rax :movqImmReg
+          /rdi :movqImmOOBReg GCSTACKMAP
+          :PAGESIZE /rsi :movqImmReg
+          < { MMAP -01 . } "!" deff
+            !PROT_READ !PROT_WRITE !PROT_EXEC bor bor /rdx :movqImmReg
+            !MAP_PRIVATE !MAP_FIXED !MAP_ANONYMOUS !MAP_GROWSDOWN bor bor bor /r10 :movqImmReg
+          > --
+          /r8 :movqImmOOBReg %FF %FF %FF %FF %FF %FF %FF %FF
+          0 /r9 :movqImmReg
+          :syscall
+
+          # TODO error handling
+          garbageCollectStack /rcx :movqImmReg
+          /rax :movqImmOOBReg GCSTACKSTART
+          /rax /rcx :movqRegMem
+          [ /rcx /r8 /r9 /r10 /r11 /r12 /r13 /r14 /r15 ] reverse { :popqReg } each
+        }
+        { sys .?freebsd } {
+          /rsp /rax :movqRegReg # FIXME: replace with real implementation
+        }
+      ] conds
+      /GCStackAllocated :jmpLbl32
+
       # run through global state and mark reachable objects
       @markAndSweep
       /r8 :pushqReg
@@ -679,6 +716,16 @@
       /r12 :pushqReg
       /r13 :pushqReg
 
+      /rsp /rcx :movqRegReg # save stack pointer
+      garbageCollectStack /rax :movqImmReg
+      /rax /rax :movqMemReg
+      /rax /rax :testqRegReg
+      /allocateGCStack :jzLbl32
+      @GCStackAllocated
+      /rax /rsp :movqRegReg
+      @useProgramStack
+      /rcx :pushqReg
+
       /r8 :movqImmOOBReg HEAPBASE # constant through mark
       /r9 :movqImmOOBReg BLOCKBASE # constant through mark
       /r10 :movqImmOOBReg MARKBASE # constant through mark
@@ -718,7 +765,8 @@
 
       # start from stack and mark all reachable blocks
       :STACKBOTTOMMARKER /rcx :movqImmReg
-      /rsp /rsi :movqRegReg
+      /rsi :popqReg # load original stack pointer
+      /rsi :pushqReg
       /rcx /rsi :cmpqRegMem
       /dataStackEmpty :jzLbl8
       @loopThroughDataStack
@@ -775,6 +823,8 @@
       /freeLoop :loopLbl8
       @noFreeNecessary
 
+      /rsp :popqReg # switch back to program stack
+
       /r13 :popqReg
       /r12 :popqReg
       /r11 :popqReg