Compiler Tokenizer working

8 files changed, 161 insertions, 7 deletions

diff --git a/compiler/elymas.ey b/compiler/elymas.ey
new file mode 100755
index 0000000..217614f
--- /dev/null
+++ b/compiler/elymas.ey
@@ -0,0 +1,9 @@
+#!/home/drahflow/elymas/interpreter/elymas
+
+"standard.ey" include
+"elymasGlobal.ey" include
+
+0 sys .argv _ len not { "Usage: ./elymas <input file.ey>" die } rep
+              * sys .file -010 .open executeFile
+
+# vim: syn=elymas
diff --git a/compiler/elymasGlobal.ey b/compiler/elymasGlobal.ey
new file mode 100644
index 0000000..13df759
--- /dev/null
+++ b/compiler/elymasGlobal.ey
@@ -0,0 +1,47 @@
+0 /TOKINT defv
+1 /TOKSTR defv
+2 /TOKID defv
+
+{ /f deff _1 /s defv regex { f } { s } ? * } /rxparse deff
+
+{ " " cat
+  { < /type defv /value defv > } /token deff
+  [ -01 { _ "" streq not } {
+    0 /any defv { /f deff any { - } { { 1 =any f } rxparse } ? * } /parse deff
+
+    "^ (.*)" { } parse
+    "^#" { "" } parse
+    "^(\\d+) +(.*)" { TOKINT token -01 } parse
+    "\"(.*)" {
+      "" /str defv
+      { _ "^\"(.*)" regex { --0 0 } { 1 } ? * } {
+        0 /strany defv { /f deff strany { - } { { 1 =strany f } rxparse } ? * } /strparse deff
+
+        "^\\\\\\\\(.*)" { str "\\" cat =str } strparse
+        "^\\\\n(.*)" { str "\n" cat =str } strparse
+        "^\\\\\"(.*)" { str "\"" cat =str } strparse
+        "^([^\"\\\\])(.*)" { str -01 cat =str } strparse
+        strany not { "Tokenization of string-liked failed" die } rep
+      } loop
+      str TOKSTR token -01
+    } parse
+    "^([^a-zA-Z ]+)([a-zA-Z]+) +(.*)" { -201 TOKSTR token " " -1203 cat cat } parse
+    "^([a-zA-Z]+|[^a-zA-Z ]+) +(.*)" { TOKID token -01 } parse
+    any not { "Tokenization failed" die } rep
+  } loop - ]
+} /tokenize deff
+
+{
+  dump
+} /interpretToken deff
+
+{ /input defv
+  "" {
+    4096 input .readstr cat
+    _ "" streq not
+  } {
+    { _ "([^\\n]*)\\n(.*)" regex } { ---10 tokenize |interpretToken each } loop
+  } loop
+} /executeFile deff
+
+# vim: syn=elymas
diff --git a/compiler/standard.ey b/compiler/standard.ey
new file mode 100644
index 0000000..7c92e35
--- /dev/null
+++ b/compiler/standard.ey
@@ -0,0 +1,7 @@
+{ "}" | *
+  { /f deff /x defv
+    { x f }
+  } quoted { } { * } ? *
+} "}0" defq
+
+# vim: syn=elymas
diff --git a/examples/working/include.ey b/examples/working/include.ey
new file mode 100644
index 0000000..4513db1
--- /dev/null
+++ b/examples/working/include.ey
@@ -0,0 +1,8 @@
+1 0 add
+{ -- 0 } /add deff
+
+"included" dump
+
+{
+  "include.ey" include
+} rep
diff --git a/examples/working/regex.ey b/examples/working/regex.ey
new file mode 100644
index 0000000..bbf557f
--- /dev/null
+++ b/examples/working/regex.ey
@@ -0,0 +1,2 @@
+"the quick brown fox jumps over the lazy dog"
+{ "(the)(.*)" regex } { dump } loop
diff --git a/interpreter/Elymas.pm b/interpreter/Elymas.pm
index 22199b7..b58bffd 100644
--- a/interpreter/Elymas.pm
+++ b/interpreter/Elymas.pm
@@ -7,7 +7,7 @@ require Exporter;
 our @ISA = qw(Exporter);
 our @EXPORT = qw(
   popInt popString popArray enstruct arrayAccess $quoted @globalCallStack
-  interpretCode execute executeString executeFile resolve canCastTo
+  interpretCode execute executeString executeFile resolve canCastTo typeEqual
 );
 
 use Data::Dumper;
@@ -114,8 +114,21 @@ sub typeEqual {
       return 0 unless @{$a->[2]} == grep { typeEqual($a->[2]->[$_], $b->[2]->[$_]) } 0 .. $#{$a->[2]};
       return 0 unless @{$a->[3]} == grep { typeEqual($a->[3]->[$_], $b->[3]->[$_]) } 0 .. $#{$a->[3]};
       return 1;
+    } elsif($a->[0] eq 'struct') {
+      return 0 unless $b->[0] eq 'struct';
+
+      my @aKeys = sort keys %{$a->[1]};
+      my @bKeys = sort keys %{$b->[1]};
+
+      return 0 unless @aKeys == @bKeys;
+      foreach my $i (0 .. $#aKeys) {
+        return 0 unless $aKeys[$i] eq $bKeys[$i];
+        return 0 unless typeEqual($a->[1]->{$aKeys[$i]}->[0], $b->[1]->{$bKeys[$i]}->[0]);
+      }
+
+      return 1;
     } else {
-      die "not yet implemented";
+      die "not yet implemented" . Dumper($a);
     }
   }
 
diff --git a/interpreter/ElymasGlobal.pm b/interpreter/ElymasGlobal.pm
index f6b45c5..81da6f0 100644
--- a/interpreter/ElymasGlobal.pm
+++ b/interpreter/ElymasGlobal.pm
@@ -99,9 +99,8 @@ our $global = {
         last if($t->[1] eq 'tok' and $t->[0] eq '[');
 
         if($type) {
-          # FIXME: This is a hack until function types work
           if(ref($type) ne 'ARRAY' or $type->[0] ne 'func') {
-            die "mismatched types in array: " . Dumper($type) if($type ne $t->[1]);
+            die "mismatched types in array: " . Dumper($type) unless typeEqual($type, $t->[1]);
           }
         } else {
           $type = $t->[1];
@@ -244,6 +243,56 @@ our $global = {
 
       push @$data, ($p->[1] eq 'int' and $p->[0] == 0? $b: $a);
     }, ['func', '?'], 'active'],
+  'include' => [sub {
+      my ($data, $scope) = @_;
+
+      my $s = popString($data);
+
+      executeFile($s, $data, $scope);
+    }, ['func', 'include'], 'active'],
+  'regex' => [sub {
+      my ($data, $scope) = @_;
+
+      my $rx = popString($data);
+      my $s = popString($data);
+
+      my @result = $s =~ qr($rx)s;
+      if(not @result) {
+        push @$data, [0, 'int'];
+      } elsif(($result[0] & ~ $result[0]) eq "0") {
+        push @$data, [1, 'int'];
+      } else {
+        foreach my $m (reverse @result) {
+          push @$data, [$m, 'string'];
+        }
+        push @$data, [1, 'int'];
+      }
+    }, ['func', 'regex'], 'active'],
+  'cat' => [sub {
+      my ($data, $scope) = @_;
+
+      my $b = pop @$data or die "Stack underflow";
+      my $a = pop @$data or die "Stack underflow";
+
+      if(ref($a->[1]) eq 'ARRAY' and $a->[1]->[0] eq 'array') {
+        if(ref($b->[1]) eq 'ARRAY' and $b->[1]->[0] eq 'array') {
+          die "Array types don't match in cat: " . Dumper($a->[1]->[3], $b->[1]->[3])
+            unless typeEqual($a->[1]->[3], $b->[1]->[3]);
+          my @res = (@$a, @$b);
+          push @$data, [\@res, ['array', 'from cat', [['range', 0, $#res]], [$a->[1]->[3]]]];
+        } else {
+          die "Mismatch between string and array in cat";
+        }
+      } elsif($a->[1] eq 'string') {
+        if($b->[1] eq 'string') {
+          push @$data, [$a->[0] . $b->[0], 'string'];
+        } else {
+          die "Mismatch between string and array in cat";
+        }
+      } else {
+        die "Neither string nor array: " . Dumper($a);
+      }
+    }, ['func', 'cat'], 'active'],
 
 # not really part of the spec, this is just for debugging
   'dump' => [sub {
@@ -274,9 +323,13 @@ our $global = {
       my ($data, $scope) = @_;
 
       my $a = pop @$data or die "Stack underflow";
-      die "Not array: " . Dumper($a) unless ref($a->[1]) eq 'ARRAY' and $a->[1]->[0] eq 'array';
-
-      push @$data, [scalar @{$a->[0]}, 'int'];
+      if(ref($a->[1]) eq 'ARRAY' and $a->[1]->[0] eq 'array') {
+        push @$data, [scalar @{$a->[0]}, 'int'];
+      } elsif(ref($a->[1]) eq 'ARRAY' and $a->[1]->[0] eq 'string') {
+        push @$data, [length $a->[0], 'int'];
+      } else {
+        die "Neither string nor array: " . Dumper($a);
+      }
     }, ['func', 'len'], 'active'],
   '=[]' => [sub {
       my ($data, $scope) = @_;
diff --git a/interpreter/ElymasSys.pm b/interpreter/ElymasSys.pm
index 09e3f71..7e6d1b9 100644
--- a/interpreter/ElymasSys.pm
+++ b/interpreter/ElymasSys.pm
@@ -18,6 +18,7 @@ our $sys = {
   'in' => [enstruct(createFile(0, &POSIX::O_RDONLY)), 'passive'],
   'out' => [enstruct(createFile(1, &POSIX::O_WRONLY)), 'passive'],
   'err' => [enstruct(createFile(2, &POSIX::O_WRONLY)), 'passive'],
+  'argv' => [[map { [$_, 'string'] } @ARGV[1 .. $#ARGV]], ['array', 'sys .argv', ['range', 0, $#ARGV - 1], ['string']], 'passive'],
 };
 
 sub createFile {
@@ -91,6 +92,20 @@ sub createFile {
 
         push @$data, [$buf, ['array', '[]', [['range', 0, $#{$buf}]], ['int']]];
       }, ['func', 'sys .file .read'], 'active'],
+    'readstr' => [sub {
+        # FIXME: give the file an encoding and respect it here, buffering half-characters if needed
+        my ($data) = @_;
+
+        die "file not open" if $$scope->{' fd'}->[0] == -1;
+
+        my $count = popInt($data);
+
+        my $buf;
+        my $ret = POSIX::read($$scope->{' fd'}->[0], $buf, $count);
+        die "read failed: $!" unless defined $ret;
+
+        push @$data, [$buf, 'string'];
+      }, ['func', 'sys .file .writestr'], 'active'],
     'write' => [sub {
         my ($data) = @_;