From 2002d5de8dd1bf33393546d0449130634e02c619 Mon Sep 17 00:00:00 2001 From: Marshall Lochbaum Date: Sun, 8 Aug 2021 20:45:44 -0400 Subject: Remove separators around : and ; during tokenization --- src/c.bqn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/c.bqn b/src/c.bqn index d5d81db9..16ad837f 100644 --- a/src/c.bqn +++ b/src/c.bqn @@ -85,7 +85,7 @@ Tokenize←{System‿vars←𝕨 t↩(w∨wf)/(vars≠⊸↓∾j++`vd»kk←≠¨k)⌾(ki⊸⊏)t # Add IDs; remove words/whitespace t-↩t(M×-⟜⊑)bS # Separators are equivalent p←≠`1¨sb←¯1↓1↓/1(∾≠∾˜)t=sep # Separator group boundaries (excludes leading and trailing) - sk←sb/˜p>∨⟜«(p+(sb-p)⊏t)∊3‿5+⊑bB # Keep the first of each group that's not just inside a bracket + sk←sb/˜p>∨⟜«(M⟜bH∨(3‿5+⊑bB)∊˜p⊸+)(sb-p)⊏t # Keep the first of each group that's not just inside a bracket t{is/˜↩𝕨⋄ie/˜↩𝕨⋄𝕨/𝕩}˜↩1¨⌾(sk⊸⊏)t≠sep # Remove the rest im←(t=bR)∨t M vd≍+´2↑kk # Identifier (or 𝕣) mask r←ir⌾(im⊸/)(vd⌊t)⊏charRole∾0 # Role -- cgit v1.2.3