a

docs/default.nix

@@ -0,0 +1,15 @@
+{ stdenvNoCC, nix-gitignore, sphinx, furo }:
+
+stdenvNoCC.mkDerivation {
+  name = "agtest-docs";
+  src = nix-gitignore.gitignoreSource [ ../.gitignore ] ./..;
+
+  buildInputs = [ sphinx furo ];
+  buildPhase = ''
+    make -C docs html
+  '';
+
+  installPhase = ''
+    cp -r docs/_build/html $out
+  '';
+}

flake.nix

@@ -13,11 +13,14 @@
 
     myPkgs = rec {
       agtest = pythonPkgs.callPackage ./. {};
+      agtest-docs = pythonPkgs.callPackage ./docs {};
     };
   in
-  {
+  rec {
     packages = flake-utils.lib.flattenTree myPkgs;
 
+    defaultPackage = packages.agtest;
+
     devShell = pkgs.mkShell {
       packages = with pythonPkgs; [
         black

src/agtest/ast.py

@@ -5,6 +5,7 @@ from re import Pattern
 
 
 def unescape(s: str) -> str:
+    """ Extracts the contents out of a quoted string """
     q = s[0]
     t = ""
     i = 1

src/agtest/gen.py

@@ -4,12 +4,13 @@ import re
 import copy
 import json
 import sys
+import copy
 from collections import defaultdict
 
 from agtest.ast import *
 
 
-def eprint(*args, **kwargs):
+def eprint(*args: Any, **kwargs: Any) -> None:
     print(*args, file=sys.stderr, **kwargs)
 
 
@@ -33,10 +34,11 @@ class NodeDesc:
 
 
 class ParseEquation:
-    def __init__(self, name: str, syms: List[str], pyty: str, pycode: str):
+    def __init__(self, name: str, syms: List[str], pyty: str, assigns: List[str], pycode: str):
         self.name = name
         self.syms = syms
         self.pyty = pyty
+        self.assigns = assigns
         self.pycode = pycode
 
 
@@ -74,15 +76,23 @@ class GenResult:
             s.append(code)
 
             for equation in rules:
+                assigns = []
+                for name, value in equation.assigns:
+                    assigns.append(f"res.{name} = {value}")
+                assigns_str = "\n".join(assigns)
+                assigns_str = textwrap.indent(assigns_str, "    ")
+
+                pycode = textwrap.indent(equation.pycode, "    ")
                 code = textwrap.dedent(
-                    f"""
-                    def {equation.name}(self, items: Any) -> Thunk[{equation.pyty}]:
-                        def inner() -> {equation.pyty}:
-                            res = {equation.pyty}()
-                            return res
-                        return Thunk(inner)
-                """
+                    """
+                    def {equation.name}(self, items: Any) -> {equation.pyty}:
+                        res = {equation.pyty}()
+                    {pycode}
+                    {assigns_str}
+                        return res
+                    """
                 )
+                code = code.format(equation=equation, assigns_str=assigns_str, pycode=pycode)
                 s.append(code)
         if not s:
             s = ["pass"]
@@ -200,7 +210,7 @@ class GenResult:
             class_fields = []
             for field_name, field_ty in self.what_fields[node_desc.name].items():
                 class_fields.append(f"{field_name}: Thunk[{field_ty}]")
-            g = textwrap.indent("\n".join(class_fields), "    ")
+            class_fields_str = textwrap.indent("\n".join(class_fields), "    ")
 
             class_decl = textwrap.dedent(
                 """
@@ -208,13 +218,45 @@ class GenResult:
                 {g}
                     pass
                 """
-            ).format(nonterminal=node_desc.nonterminal, g=g)
+            ).format(nonterminal=node_desc.nonterminal, g=class_fields_str)
             self.extra += class_decl
 
             # print(node_desc.name, node_desc.node.ifaces)
 
             for variant in node_desc.node.variants:
                 v_class_name = gensym(f"{node_desc.nonterminal}_var")
+
+                prod_name = gensym(node_desc.nonterminal + "_")
+
+                # figure out which of the symbols are inputs (aka contain data)
+                # vs. literals which only parse
+                # TODO: probably should also just make literals return their values
+                seq = []
+                inputs = []
+                renamed = dict()
+                for i, sym in enumerate(variant.prod):
+                    isInput, n = self._resolve_production(sym)
+                    var_name = f"inp_{n}_idx{i}"
+
+                    if isInput:
+                        inputs.append((i, n, var_name))
+                    seq.append(n)
+
+                    if isinstance(sym, SymRename):
+                        renamed[sym.name] = i
+
+                for field in class_fields:
+                    eprint("FIELD", field)
+
+                # generate assignments for codegen
+                input_fields = copy.deepcopy(class_fields)
+                assigns = []
+                for i, inp, var_name in inputs:
+                    name = gensym(f"{v_class_name}_inp")
+                    input_fields.append(f"{var_name}: Any = 0")
+                    assigns.append((var_name, f"items[{i}]"))
+                class_fields_str = textwrap.indent("\n".join(input_fields), "    ")
+
                 class_decl = textwrap.dedent(
                     """
                     class {v_class_name}({nonterminal}):
@@ -222,25 +264,13 @@ class GenResult:
                         pass
                     """
                 ).format(
-                    v_class_name=v_class_name, nonterminal=node_desc.nonterminal, g=g
+                    v_class_name=v_class_name, nonterminal=node_desc.nonterminal, g=class_fields_str
                 )
                 self.extra += class_decl
 
-                prod_name = gensym(node_desc.nonterminal + "_")
-                # print("PRODUCTION", prod_name, variant.prod)
-
-                seq = []
-                inputs = []
-                for i, sym in enumerate(variant.prod):
-                    isInput, n = self._resolve_production(sym)
-                    if isInput:
-                        inputs.append((i, n))
-                    seq.append(n)
-                eprint("INPUTS", node_desc.nonterminal, inputs)
-
                 pycode = ""
                 self.parse_rules[node_desc.nonterminal].append(
-                    ParseEquation(prod_name, seq, v_class_name, pycode)
+                    ParseEquation(prod_name, seq, v_class_name, assigns, pycode)
                 )
 
                 # create an environment for checking the equations based on the