Handle character and string literals in Tokenize

1 files changed, 14 insertions, 7 deletions

diff --git a/c.bqn b/c.bqn
index 81ae0bdb..444e481f 100755
--- a/c.bqn
+++ b/c.bqn
@@ -10,7 +10,6 @@ Enc2←{𝕨⥊2(|∾˜(𝕨-1)Enc2⌊∘÷˜)⍟(0<𝕨)𝕩}
 
 lf‿tab←•UCS 10‿9
 charSet←∾charGroups←⟨
-  "⍝"                   ⍝ Remark
   •d ⍝ +⟜(↕10)⌾•UCS'0'  ⍝ Digit
   "¯.π∞"                ⍝ Numeric
   "_"∾' '(⊢∾+)⌾•UCS•a   ⍝ Alphabetic
@@ -23,20 +22,28 @@ charSet←∾charGroups←⟨
   "←↩→"                 ⍝ Gets
   "(){}⟨⟩"              ⍝ Bracket
   "‿"                   ⍝ Ligature
-  "'"""                 ⍝ Quote
+⍝ ⍝'" eliminated during tokenization
 ⟩
-bR‿bD‿bN‿bA‿bF‿bM‿bC‿bP‿bW‿bS‿bG‿bB‿bL‿bQ←↕∘≠⌾∾charGroups
+bD‿bN‿bA‿bF‿bM‿bC‿bP‿bW‿bS‿bG‿bB‿bL←↕∘≠⌾∾charGroups
 vi←≠charSet
 
 Tokenize←{
-  args←charSet⊸⊐¨𝕨⋄c←charSet⊐𝕩
-  c(/˜)↩¬<○(⌈`↕∘≠⊸×)´<˘((⊑bS)∾bR)=⌜c
+  r←𝕩='⍝'⋄s←/(≠↑2⊸↓)⊸∧𝕩=⊑"'"⋄d←/𝕩='"'
+  g←⍋q←∾⟨  s⋄¯1↓d⋄/r⟩ ⋄q↩g⊏q
+  e← g⊏∾⟨2+s⋄ 1↓d⋄(⊢-¯1↓0∾⊢)∘⊏⟜(0∾+`r)⊸//(𝕩=lf)∾1⟩
+  Se←{(⊏˜𝕨)Se 1¨⌾((𝕩/𝕨)⊸⊏)𝕩}⍟{0=⊑⌽𝕩}
+  st←¯1↓Se⟜(1↑˜≠)∾⟜≠q⍋e⋄b←st/q∾˘e
+  ToI←(≠𝕩)↑/⁼∘⥊⋄f←¬≠`ToI b
+  cb←(¬(st/q)⊏r)/b
+  nl←≠lu←⍷lit←𝕩⊔˜1-˜(+`ToI⊑˘cb)×≠`ToI cb
+  cl←f/ToI⊑∘⌽˘cb
+  args←charSet⊸⊐¨𝕨⋄c←charSet⊐f/𝕩
   w←(<´≠↕0∾⊢)l←c∊∾bD‿bN‿bA
   id←⍷args∾(bA∊˜w/c)⍒⊸⊏i←(1-˜l×+`w)⊔c
   nv←+´bA∊˜⊑¨id
-  c↩(w∨¬l∨c∊bW)/(vi+id⊐i)⌾(w⊸/)c
+  c↩(w∨¬l∨c∊bW)/(vi+id⊐i)⌾(w⊸/)(vi+(≠id)+lu⊐lit)⌾(cl⊸/)c
   c(/˜)↩¬(1⌾⊑1⌽1⌾⊑)⊸∧c∊bS
-  ⟨c,nv,nv-≠args,nv↓id⟩
+  ⟨c,nv,nv-≠args,(nv↓id)∾lu⟩
 }
 
 Parse←{