<
  { ==f =*re _ ==s re f { s } ? * } /rxparse deffd

  { ==TOKID ==TOKSTR ==TOKINT ==TOKFLOAT
    " " cat
    { < /handle deff /value defv > } /token deff
    [ -01 { _ "" eq not } {
      0 ==matched { =*f matched { -- } { { 1 =matched f } rxparse } ? * } /parse deffst

      { "^ +(.*)" regex } { } parse
      { "^#" regex } { "" } parse
      { "^(\\d+) +(.*)" regex } { TOKINT token -01 } parse
      { "^(\\d[0-9.]*([eE]-?\\d+)?) +(.*)" regex } { TOKFLOAT token -02 } parse
      { "^\"(.*)" regex } {
        "" /str defv
        { _ "^\"(.*)" regex { -01 -- 0 } { 1 } ? * } {
          0 /strmatched defv { /f deff strmatched { -- } { { 1 =strmatched f } rxparse } ? * } /strparse deff

          { "^\\\\\\\\(.*)" regex } { str "\\" cat =str } strparse
          { "^\\\\e(.*)" regex } { str "\e" cat =str } strparse
          { "^\\\\n(.*)" regex } { str "\n" cat =str } strparse
          { "^\\\\r(.*)" regex } { str "\r" cat =str } strparse
          { "^\\\\0(.*)" regex } { str "\0" cat =str } strparse
          { "^\\\\\"(.*)" regex } { str "\"" cat =str } strparse
          { "^([^\"\\\\])(.*)" regex } { str -01 cat =str } strparse
          strmatched not { "Tokenization of string-like failed" die } rep
        } loop
        str TOKSTR token -01
      } parse
      { "^([^a-zA-Z0-9 ]+)([a-zA-Z0-9][^ ]*) +(.*)" regex } { -201 TOKSTR token " " -1203 cat cat } parse
      { "^([a-zA-Z0-9]+|[^a-zA-Z0-9 ]+) +(.*)" regex } { TOKID token -01 } parse

      matched not { "Tokenization failed: " -01 cat die } rep
    } loop -- ]
  } /tokenize deff
> /elymas defv

# vim: syn=elymas