From 7c539384346b0a05f558969f689a779c90ba6bfa Mon Sep 17 00:00:00 2001
From: Marshall Lochbaum <mwlochbaum@gmail.com>
Date: Sun, 25 Oct 2020 15:16:29 -0400
Subject: Return all multi-character token values (including identifiers) from
 Tokenize

---
 src/c.bqn | 8 ++++----
 wc.bqn    | 3 ++-
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/c.bqn b/src/c.bqn
index 1d1332e5..19ebd508 100644
--- a/src/c.bqn
+++ b/src/c.bqn
@@ -69,7 +69,7 @@ Tokenize←{
   c/˜↩¬(1«c∊3‿5+⊑bB)∧c=sep                  # ...and trailing ones. In sequence for repeated trailing.
   cr←ir⌾((c M vi∾≠⊑k)⊸/)(vi⌊c)⊏charRole∾0   # Role
   c+↩5×c M⟨⊑bI,5⟩                           # Case-insensitive special names
-  ⟨c,cr,≠⊑k,∾1↓k⟩
+  ⟨c,cr,k⟩
 }
 
 # 𝕩 is a list of tokens that contains the numeric literals, each
@@ -194,7 +194,7 @@ LEB←{
 }
 
 Compile←{
-  ⟨t,r,nVar,const⟩←Tokenize𝕩
-  ⟨bc,pr,blk⟩←r‿nVar Parse t
-  ⟨bc,(pr⊏𝕨)∾const,<˘⍉>blk⟩
+  ⟨tok,role,val⟩←Tokenize 𝕩
+  ⟨bc,prim,blk⟩←⟨role,≠⊑val⟩ Parse tok
+  ⟨bc, ∾⟨prim⊏𝕨⟩∾1↓val, <˘⍉>blk⟩
 }
diff --git a/wc.bqn b/wc.bqn
index 3c5b3090..95dd7720 100644
--- a/wc.bqn
+++ b/wc.bqn
@@ -47,7 +47,8 @@ fntab←⍉(0¨tab1)∾tab1≍tab2
 f64←127-3
 
 GenFn←{
-  ⟨t,r,nVar,lits⟩←Tokenize𝕩
+  ⟨t,r,k⟩←Tokenize𝕩
+  nVar←≠⊑k⋄lits←∾1↓k
   nLoc←nVar-𝕨⊢0
   t⊏˜↩⍋+`-˝(2‿3+⊑bB)=⌜t
   nd←+´c←t=3+⊑bB
-- 
cgit v1.2.3