blob: 3cc1c9766827e276f79c2b43d6b14932fa807531 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
|
<
{ ==f =*re _ ==s re f { s } ? * } /rxparse deffd
{ ==TOKID ==TOKSTR ==TOKINT ==TOKFLOAT
" " cat
{ < /handle deff /value defv > } /token deff
[ -01 { _ "" eq not } {
0 ==matched { =*f matched { -- } { { 1 =matched f } rxparse } ? * } /parse deffst
{ "^ +(.*)" regex } { } parse
{ "^#" regex } { "" } parse
{ "^(\\d+) +(.*)" regex } { TOKINT token -01 } parse
{ "^(\\d[0-9.]*([eE]-?\\d+)?) +(.*)" regex } { TOKFLOAT token -02 } parse
{ "^\"(.*)" regex } {
"" /str defv
{ _ "^\"(.*)" regex { -01 -- 0 } { 1 } ? * } {
0 /strmatched defv { /f deff strmatched { -- } { { 1 =strmatched f } rxparse } ? * } /strparse deff
{ "^\\\\\\\\(.*)" regex } { str "\\" cat =str } strparse
{ "^\\\\e(.*)" regex } { str "\e" cat =str } strparse
{ "^\\\\n(.*)" regex } { str "\n" cat =str } strparse
{ "^\\\\r(.*)" regex } { str "\r" cat =str } strparse
{ "^\\\\0(.*)" regex } { str "\0" cat =str } strparse
{ "^\\\\\"(.*)" regex } { str "\"" cat =str } strparse
{ "^([^\"\\\\])(.*)" regex } { str -01 cat =str } strparse
strmatched not { "Tokenization of string-like failed" die } rep
} loop
str TOKSTR token -01
} parse
{ "^([^a-zA-Z0-9 ]+)([a-zA-Z0-9][^ ]*) +(.*)" regex } { -201 TOKSTR token " " -1203 cat cat } parse
{ "^([a-zA-Z0-9]+|[^a-zA-Z0-9 ]+) +(.*)" regex } { TOKID token -01 } parse
matched not { "Tokenization failed: " -01 cat die } rep
} loop -- ]
} /tokenize deff
> /elymas defv
# vim: syn=elymas
|
