aboutsummaryrefslogtreecommitdiff
path: root/src/c.bqn
diff options
context:
space:
mode:
authorMarshall Lochbaum <mwlochbaum@gmail.com>2021-08-08 20:45:44 -0400
committerMarshall Lochbaum <mwlochbaum@gmail.com>2021-08-08 20:47:16 -0400
commit2002d5de8dd1bf33393546d0449130634e02c619 (patch)
tree500b63c59317dd2918bbfd2e2e9d332962f99979 /src/c.bqn
parentb4f17f5846c7621062b38c55106ba0767115ce57 (diff)
Remove separators around : and ; during tokenization
Diffstat (limited to 'src/c.bqn')
-rw-r--r--src/c.bqn2
1 files changed, 1 insertions, 1 deletions
diff --git a/src/c.bqn b/src/c.bqn
index d5d81db9..16ad837f 100644
--- a/src/c.bqn
+++ b/src/c.bqn
@@ -85,7 +85,7 @@ Tokenize←{System‿vars←𝕨
t↩(w∨wf)/(vars≠⊸↓∾j++`vd»kk←≠¨k)⌾(ki⊸⊏)t # Add IDs; remove words/whitespace
t-↩t(M×-⟜⊑)bS # Separators are equivalent
p←≠`1¨sb←¯1↓1↓/1(∾≠∾˜)t=sep # Separator group boundaries (excludes leading and trailing)
- sk←sb/˜p>∨⟜«(p+(sb-p)⊏t)∊3‿5+⊑bB # Keep the first of each group that's not just inside a bracket
+ sk←sb/˜p>∨⟜«(M⟜bH∨(3‿5+⊑bB)∊˜p⊸+)(sb-p)⊏t # Keep the first of each group that's not just inside a bracket
t{is/˜↩𝕨⋄ie/˜↩𝕨⋄𝕨/𝕩}˜↩1¨⌾(sk⊸⊏)t≠sep # Remove the rest
im←(t=bR)∨t M vd≍+´2↑kk # Identifier (or 𝕣) mask
r←ir⌾(im⊸/)(vd⌊t)⊏charRole∾0 # Role