From 58bd5f088959c81e7a25aaf63de8fa14d1e0214a Mon Sep 17 00:00:00 2001
From: "Johan B.W. de Vries" <info@jbwdevries.nl>
Date: Mon, 5 Apr 2021 10:41:26 +0200
Subject: [PATCH] More cleanup [skip-ci]

---
 Makefile                      |  10 +-
 compile.py                    |  30 ----
 py2wasm/__main__.py           |  25 +++
 py2wasm/python.py             | 311 +++++++++++++++++++++++-----------
 py2wasm/utils.py              |  18 ++
 py2wasm/wasm.py               |  74 ++++++--
 pylintrc                      |   2 +
 requirements.txt              |   4 +-
 tests/integration/helpers.py  |  77 +++++++++
 tests/integration/test_fib.py |  79 +--------
 10 files changed, 409 insertions(+), 221 deletions(-)
 delete mode 100644 compile.py
 create mode 100644 py2wasm/__main__.py
 create mode 100644 py2wasm/utils.py
 create mode 100644 pylintrc
 create mode 100644 tests/integration/helpers.py

diff --git a/Makefile b/Makefile
index 3ed270c..3633841 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
-%.wat: %.py compile.py
-	python3.8 compile.py $< $@
+%.wat: %.py py2wasm/*.py
+	python3.8 -m py2wasm $< $@
 
 %.wasm: %.wat
 	wat2wasm $^ -o $@
@@ -10,6 +10,12 @@ server:
 test: venv/.done
 	venv/bin/pytest tests
 
+lint: venv/.done
+	venv/bin/pylint py2wasm
+
+typecheck: venv/.done
+	venv/bin/mypy --strict py2wasm
+
 venv/.done: requirements.txt
 	python3.8 -m venv venv
 	venv/bin/python3 -m pip install -r $^
diff --git a/compile.py b/compile.py
deleted file mode 100644
index 50a6168..0000000
--- a/compile.py
+++ /dev/null
@@ -1,30 +0,0 @@
-import _ast
-import ast
-import sys
-
-from py2wasm.wasm import Module
-from py2wasm.python import Visitor
-
-def process(input: str, input_name: str) -> str:
-    res = ast.parse(input, input_name)
-
-    module = Module()
-
-    visitor = Visitor(module)
-    visitor.visit(res)
-
-    return module.generate()
-
-def main(source: str, sink: str) -> int:
-    with open(source, 'r') as fil:
-        code_py = fil.read()
-
-    code_wat = process(code_py, source)
-
-    with open(sink, 'w') as fil:
-        fil.write(code_wat)
-
-    return 0
-
-if __name__ == '__main__':
-    sys.exit(main(*sys.argv[1:]))
diff --git a/py2wasm/__main__.py b/py2wasm/__main__.py
new file mode 100644
index 0000000..59757dd
--- /dev/null
+++ b/py2wasm/__main__.py
@@ -0,0 +1,25 @@
+"""
+Functions for using this module from CLI
+"""
+
+import sys
+
+from .utils import process
+
+def main(source: str, sink: str) -> int:
+    """
+    Main method
+    """
+
+    with open(source, 'r') as fil:
+        code_py = fil.read()
+
+    code_wat = process(code_py, source)
+
+    with open(sink, 'w') as fil:
+        fil.write(code_wat)
+
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main(*sys.argv[1:])) # pylint: disable=E1120
diff --git a/py2wasm/python.py b/py2wasm/python.py
index 7566cbf..da7c140 100644
--- a/py2wasm/python.py
+++ b/py2wasm/python.py
@@ -1,11 +1,134 @@
+"""
+Code for parsing the source (python-alike) code
+"""
+
+from typing import List, Optional, Union, Tuple
+
 import ast
 
-from .wasm import Function, Import, Module, Statement
+from . import wasm
+
+# pylint: disable=C0103,R0201
+
+class Visitor:
+    """
+    Class to visit a Python syntax tree
+
+    Since we need to visit the whole tree, there's no point in subclassing
+    the buildin visitor
+    """
+    def visit_Module(self, node: ast.Module) -> wasm.Module:
+        """
+        Visits a Python module, which results in a wasm Module
+        """
+        assert not node.type_ignores
+
+        module = wasm.Module()
+
+        for stmt in node.body:
+            if isinstance(stmt, ast.FunctionDef):
+                wnode = self.visit_FunctionDef(stmt)
+
+                if isinstance(wnode, wasm.Import):
+                    module.imports.append(wnode)
+                else:
+                    module.functions.append(wnode)
+                continue
+
+            raise NotImplementedError(stmt)
+
+        return module
+
+    def visit_FunctionDef(self, node: ast.FunctionDef) -> Union[wasm.Import, wasm.Function]:
+        """
+        A Python function definition with the @external decorator is returned as import.
+
+        Other functions are returned as (exported) functions.
+        """
+
+        exported = False
+
+        if node.decorator_list:
+            assert 1 == len(node.decorator_list)
+
+            decorator = node.decorator_list[0]
+            if isinstance(decorator, ast.Name):
+                assert 'exported' == decorator.id
+                exported = True
+            elif isinstance(decorator, ast.Call):
+                call = decorator
+
+                assert not node.type_comment
+                assert 1 == len(node.body)
+                assert isinstance(node.body[0], ast.Expr)
+                assert isinstance(node.body[0].value, ast.Ellipsis)
+
+                assert isinstance(call.func, ast.Name)
+                assert 'imported' == call.func.id
+
+                assert not call.keywords
+
+                module, name, intname = _parse_import_decorator(node.name, call.args)
+
+                return wasm.Import(module, name, intname, _parse_import_args(node.args))
+
+        return wasm.Function(node.name, exported)
+
+def _parse_import_decorator(func_name: str, args: List[ast.expr]) -> Tuple[str, str, str]:
+    """
+    Parses an @import decorator
+    """
+
+    assert 0 < len(args) < 3
+    str_args = [
+        arg.value
+        for arg in args
+        if isinstance(arg, ast.Constant)
+        and isinstance(arg.value, str)
+    ]
+    assert len(str_args) == len(args)
+
+    module = str_args.pop(0)
+    if str_args:
+        name = str_args.pop(0)
+    else:
+        name = func_name
+
+    return module, name, func_name
+
+def _parse_import_args(args: ast.arguments) -> List[Tuple[str, str]]:
+    """
+    Parses the arguments for an @imported method
+    """
+
+    assert not args.vararg
+    assert not args.kwonlyargs
+    assert not args.kw_defaults
+    assert not args.kwarg
+    assert not args.defaults # Maybe support this later on
+
+    arg_list = [
+        *args.posonlyargs,
+        *args.args,
+    ]
+
+    return [
+        (arg.arg, _parse_annotation(arg.annotation))
+        for arg in arg_list
+    ]
+
+def _parse_annotation(ann: Optional[ast.expr]) -> str:
+    """
+    Parses a typing annotation
+    """
+    assert ann is not None, 'Web Assembly requires type annotations'
+    assert isinstance(ann, ast.Name)
+    result = ann.id
+    assert result in ['i32']
+    return result
+
+
 
-class Visitor(ast.NodeVisitor):
-    def __init__(self, module: Module) -> None:
-        self._stack = []
-        self.module = module
 
     # def visit_ImportFrom(self, node):
     #     for alias in node.names:
@@ -14,96 +137,88 @@ class Visitor(ast.NodeVisitor):
     #             alias.name,
     #             alias.asname,
     #         ))
-
-    def visit_FunctionDef(self, node):
-        is_export = False
-
-        if node.decorator_list:
-            # TODO: Support normal decorators
-            assert 1 == len(node.decorator_list)
-
-            call = node.decorator_list[0]
-            if not isinstance(call, ast.Name):
-                assert isinstance(call, ast.Call)
-
-                assert 'external' == call.func.id
-                assert 1 == len(call.args)
-                assert isinstance(call.args[0].value, str)
-
-                import_ = Import(
-                    call.args[0].value,
-                    node.name,
-                    node.name,
-                )
-
-                import_.params = [
-                    arg.annotation.id
-                    for arg in node.args.args
-                ]
-
-                self.module.imports.append(import_)
-                return
-
-            assert call.id == 'export'
-            is_export = True
-
-        func = Function(
-            node.name,
-            is_export,
-        )
-
-        for arg in node.args.args:
-            func.params.append(
-                
-            )
-
-        self._stack.append(func)
-        self.generic_visit(node)
-        self._stack.pop()
-
-        self.module.functions.append(func)
-
-    def visit_Expr(self, node):
-        self.generic_visit(node)
-
-    def visit_Call(self, node):
-        self.generic_visit(node)
-
-        func = self._stack[-1]
-        func.statements.append(
-            Statement('call', '$' + node.func.id)
-        )
-
-    def visit_BinOp(self, node):
-        self.generic_visit(node)
-
-        func = self._stack[-1]
-
-        if 'Add' == node.op.__class__.__name__:
-            func.statements.append(
-                Statement('i32.add')
-            )
-        elif 'Mult' == node.op.__class__.__name__:
-            func.statements.append(
-                Statement('i32.mul')
-            )
-        else:
-            raise NotImplementedError
-
-    def visit_Constant(self, node):
-        if not self._stack:
-            # Constant outside of any function
-            imp = self.imports[-1]
-            prefix = imp.name + '('
-            val = node.value.strip()
-
-            if val.startswith(prefix) and val.endswith(')'):
-                imp.params = val[len(prefix):-1].split(',')
-        else:
-            func = self._stack[-1]
-            if isinstance(node.value, int):
-                func.statements.append(
-                    Statement('i32.const', str(node.value))
-                )
-
-        self.generic_visit(node)
+    #
+    # def generic_visit(self, node: Any) -> None:
+    #     print(node)
+    #     super().generic_visit(node)
+    #
+    # def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
+    #     is_export = False
+    #
+    #     if node.decorator_list:
+    #         assert 1 == len(node.decorator_list)
+    #
+    #         call = node.decorator_list[0]
+    #         if not isinstance(call, ast.Name):
+    #             assert isinstance(call, ast.Call)
+    #
+    #             assert 'external' == call.func.id
+    #             assert 1 == len(call.args)
+    #             assert isinstance(call.args[0].value, str)
+    #
+    #             import_ = Import(
+    #                 call.args[0].value,
+    #                 node.name,
+    #                 node.name,
+    #             )
+    #
+    #             import_.params = [
+    #                 arg.annotation.id
+    #                 for arg in node.args.args
+    #             ]
+    #
+    #             self.module.imports.append(import_)
+    #             return
+    #
+    #         assert call.id == 'export'
+    #         is_export = True
+    #
+    #     func = Function(
+    #         node.name,
+    #         is_export,
+    #     )
+    #
+    #     for arg in node.args.args:
+    #         func.params.append(
+    #             (arg.arg, arg.annotation.id)
+    #         )
+    #     func.result = node.returns
+    #
+    #     self._stack.append(func)
+    #     self.generic_visit(node)
+    #     self._stack.pop()
+    #
+    #     self.module.functions.append(func)
+    #
+    # def visit_Call(self, node: ast.Call) -> None:
+    #     self.generic_visit(node)
+    #
+    #     func = self._stack[-1]
+    #     func.statements.append(
+    #         Statement('call', '$' + node.func.id)
+    #     )
+    #
+    # def visit_BinOp(self, node: ast.BinOp) -> None:
+    #     self.generic_visit(node)
+    #
+    #     func = self._stack[-1]
+    #
+    #     if 'Add' == node.op.__class__.__name__:
+    #         func.statements.append(
+    #             Statement('i32.add')
+    #         )
+    #     elif 'Mult' == node.op.__class__.__name__:
+    #         func.statements.append(
+    #             Statement('i32.mul')
+    #         )
+    #     else:
+    #         raise NotImplementedError
+    #
+    # def visit_Constant(self, node: ast.Constant) -> None:
+    #     func = self._stack[-1]
+    #     if isinstance(node.value, int):
+    #         func.statements.append(
+    #             Statement('i32.const', str(node.value))
+    #         )
+    #
+    #     self.generic_visit(node)
diff --git a/py2wasm/utils.py b/py2wasm/utils.py
new file mode 100644
index 0000000..c1e9b72
--- /dev/null
+++ b/py2wasm/utils.py
@@ -0,0 +1,18 @@
+"""
+Utility functions
+"""
+
+import ast
+
+from py2wasm.python import Visitor
+
+def process(source: str, input_name: str) -> str:
+    """
+    Processes the python code into web assembly code
+    """
+    res = ast.parse(source, input_name)
+
+    visitor = Visitor()
+    module = visitor.visit_Module(res)
+
+    return module.generate()
diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py
index ce6f81f..0843a28 100644
--- a/py2wasm/wasm.py
+++ b/py2wasm/wasm.py
@@ -1,45 +1,87 @@
+"""
+Python classes for storing the representation of Web Assembly code
+"""
+
+from typing import Iterable, List, Tuple
+
 class Import:
-    def __init__(self, module, name, intname):
+    """
+    Represents a Web Assembly import
+    """
+    def __init__(
+            self,
+            module: str,
+            name: str,
+            intname: str,
+            params: Iterable[Tuple[str, str]],
+        ) -> None:
         self.module = module
         self.name = name
         self.intname = intname
-        self.params = None
+        self.params = [*params]
 
-    def generate(self):
+    def generate(self) -> str:
+        """
+        Generates the text version
+        """
         return '(import "{}" "{}" (func ${}{}))'.format(
             self.module,
             self.name,
             self.intname,
-            ''.join(' (param {})'.format(x) for x in self.params)
+            ''.join(' (param {})'.format(x[1]) for x in self.params)
         )
 
 class Statement:
-    def __init__(self, name, *args):
+    """
+    Represents a Web Assembly statement
+    """
+    def __init__(self, name: str, *args: str):
         self.name = name
         self.args = args
 
-    def generate(self):
+    def generate(self) -> str:
+        """
+        Generates the text version
+        """
         return '{} {}'.format(self.name, ' '.join(self.args))
 
 class Function:
-    def __init__(self, name, exported=True):
+    """
+    Represents a Web Assembly function
+    """
+    def __init__(self, name: str, exported: bool = True) -> None:
         self.name = name
-        self.exported = exported # TODO: Use __all__!
-        self.params = []
-        self.statements = []
+        self.exported = exported
+        self.params: List[Tuple[str, str]] = []
+        self.returns = None
+        self.statements: List[Statement] = []
+
+    def generate(self) -> str:
+        """
+        Generates the text version
+        """
+        header = ('(export "{}")' if self.exported else '${}').format(self.name)
+
+        for nam, typ in self.params:
+            header += ' (param ${} {})'.format(nam, typ)
 
-    def generate(self):
         return '(func {}\n    {})'.format(
-            ('(export "{}")' if self.exported else '${}').format(self.name),
+            header,
             '\n    '.join(x.generate() for x in self.statements),
         )
 
 class Module:
-    def __init__(self, imports = None, functions = None) -> None:
-        self.imports = imports or []
-        self.functions = functions or []
+    """
+    Represents a Web Assembly module
+    """
+    def __init__(self) -> None:
+        self.imports: List[Import] = []
+        self.functions: List[Function] = []
 
-    def generate(self):
+    def generate(self) -> str:
+        """
+        Generates the text version
+        """
         return '(module\n  {}\n  {})\n'.format(
             '\n  '.join(x.generate() for x in self.imports),
             '\n  '.join(x.generate() for x in self.functions),
diff --git a/pylintrc b/pylintrc
new file mode 100644
index 0000000..72d74db
--- /dev/null
+++ b/pylintrc
@@ -0,0 +1,2 @@
+[MASTER]
+disable=C0122,R0903
diff --git a/requirements.txt b/requirements.txt
index 9e01a86..ef8699b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,4 @@
-pywasm==1.0.7
+mypy==0.812
+pylint==2.7.4
 pytest==6.2.2
+pywasm==1.0.7
diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py
new file mode 100644
index 0000000..57ed359
--- /dev/null
+++ b/tests/integration/helpers.py
@@ -0,0 +1,77 @@
+import io
+import subprocess
+import sys
+from tempfile import NamedTemporaryFile
+
+from pywasm import binary
+from pywasm import Runtime
+
+from py2wasm.utils import process
+
+def wat2wasm(code_wat):
+    with NamedTemporaryFile('w+t') as input_fp:
+        input_fp.write(code_wat)
+        input_fp.flush()
+
+        with NamedTemporaryFile('w+b') as output_fp:
+            result = subprocess.run(
+                [
+                    'wat2wasm',
+                    input_fp.name,
+                    '-o',
+                    output_fp.name,
+                ],
+                check=True,
+            )
+
+            output_fp.seek(0)
+
+            return output_fp.read()
+
+class SuiteResult:
+    def __init__(self):
+        self.log_int32_list = []
+
+    def callback_log_int32(self, store, value):
+        del store # auto passed by pywasm
+
+        self.log_int32_list.append(value)
+
+    def make_imports(self):
+        return {
+            'console': {
+                'logInt32': self.callback_log_int32,
+            }
+        }
+
+class Suite:
+    def __init__(self, code_py, test_name):
+        self.code_py = code_py
+        self.test_name = test_name
+
+    def run_code(self, *args):
+        """
+        Compiles the given python code into wasm and
+        then runs it
+
+        Returned is an object with the results set
+        """
+        code_wat = process(self.code_py, self.test_name)
+
+        line_list = code_wat.split('\n')
+        line_no_width = len(str(len(line_list)))
+        for line_no, line_txt in enumerate(line_list):
+            sys.stderr.write('{}  {}\n'.format(
+                str(line_no + 1).zfill(line_no_width),
+                line_txt,
+            ))
+
+        code_wasm = wat2wasm(code_wat)
+        module = binary.Module.from_reader(io.BytesIO(code_wasm))
+
+        result = SuiteResult()
+        runtime = Runtime(module, result.make_imports(), {})
+
+        result.returned_value = runtime.exec('testEntry', args)
+
+        return result
diff --git a/tests/integration/test_fib.py b/tests/integration/test_fib.py
index 16fa464..fa1eaf9 100644
--- a/tests/integration/test_fib.py
+++ b/tests/integration/test_fib.py
@@ -1,84 +1,15 @@
-import io
-import subprocess
-import sys
-from tempfile import NamedTemporaryFile
-
-from pywasm import binary
-from pywasm import Runtime
-
-from compile import process
-
-def wat2wasm(code_wat):
-    with NamedTemporaryFile('w+t') as input_fp:
-        input_fp.write(code_wat)
-        input_fp.flush()
-
-        with NamedTemporaryFile('w+b') as output_fp:
-            result = subprocess.run(
-                [
-                    'wat2wasm',
-                    input_fp.name,
-                    '-o',
-                    output_fp.name,
-                ],
-                check=True,
-            )
-
-            output_fp.seek(0)
-
-            return output_fp.read()
-
-class SuiteResult:
-    def __init__(self):
-        self.log_int32_list = []
-
-    def callback_log_int32(self, store, value):
-        del store # auto passed by pywasm
-
-        self.log_int32_list.append(value)
-
-    def make_imports(self):
-        return {
-            'console': {
-                'logInt32': self.callback_log_int32,
-            }
-        }
-
-class Suite:
-    def __init__(self, code_py, test_name):
-        self.code_py = code_py
-        self.test_name = test_name
-
-    def run_code(self, *args):
-        """
-        Compiles the given python code into wasm and
-        then runs it
-
-        Returned is an object with the results set
-        """
-        code_wat = process(self.code_py, self.test_name)
-        sys.stderr.write(code_wat)
-
-        code_wasm = wat2wasm(code_wat)
-        module = binary.Module.from_reader(io.BytesIO(code_wasm))
-
-        result = SuiteResult()
-        runtime = Runtime(module, result.make_imports(), {})
-
-        result.returned_value = runtime.exec('testEntry', args)
-
-        return result
+from .helpers import Suite
 
 def test_fib():
     code_py = """
-@external('console')
+@imported('console')
 def logInt32(value: i32) -> None:
     ...
 
-def helper(arg: i32) -> i32:
-    return arg + 1
+def helper(value: i32) -> i32:
+    return value + 1
 
-@export
+@exported
 def testEntry():
     logInt32(13 + 13 * helper(122))
 """