aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorMarshall Lochbaum <mwlochbaum@gmail.com>2020-10-25 20:47:28 -0400
committerMarshall Lochbaum <mwlochbaum@gmail.com>2020-10-25 20:47:28 -0400
commit8266746c221a516c654087d823e73d9ab5a1a9e1 (patch)
tree673471040b3b6e74859db52a24cc9cc0506fef2d /src
parente69bbebc7c2057b0c31bee42308c75ed8a588f55 (diff)
Remove redundant separators with only one filter
Diffstat (limited to 'src')
-rw-r--r--src/c.bqn5
1 files changed, 3 insertions, 2 deletions
diff --git a/src/c.bqn b/src/c.bqn
index 96e243b8..4c077d01 100644
--- a/src/c.bqn
+++ b/src/c.bqn
@@ -65,8 +65,9 @@ Tokenize←{
k←id‿num‿chr‿str⋄k(⊢>¯1»⌈`)⊸/¨˜↩j←⊐¨k # IDs j into uniques k
t↩(w∨¬l∨t M bW)/(∾j++`vi»≠¨k)⌾(ki⊸⊏)t # Add IDs; remove words/whitespace
t-↩t(M×-⟜⊑)bS # Separators are equivalent
- t/˜↩¬(1»(t∊2‿4+⊑bB)∨⊢)⊸∧t=sep # Remove repeated and leading separators
- t/˜↩¬(1«t∊3‿5+⊑bB)∧t=sep # ...and trailing ones. In sequence for repeated trailing.
+ p←≠`1¨sb←¯1↓1↓/1(∾≠∾˜)t=sep # Separator group boundaries (excludes leading and trailing)
+ sk←sb/˜p>∨⟜«(p+(sb-p)⊏t)∊3‿5+⊑bB # Keep the first of each group that's not just inside a bracket
+ t/˜↩1¨⌾(sk⊸⊏)t≠sep # Remove the rest
r←ir⌾((t M vi∾≠⊑k)⊸/)(vi⌊t)⊏charRole∾0 # Role
t+↩5×t M⟨⊑bI,5⟩ # Case-insensitive special names
⟨t,r,k⟩