Cleanup to helpers, making use of runners

TODO.md

@@ -2,5 +2,5 @@
 
 - Rename ourlang.Struct to OurTypeStruct
   - Maybe rename this to a types module?
-- Remove references to log_int32_list
 - Fix naming in Suite() calls
+- Replace ___new_reference___ by stdlib.alloc.__alloc__

stubs/wasmer.pyi

@@ -0,0 +1,39 @@
+from typing import Any, Dict, Callable, Union
+
+def wat2wasm(inp: str) -> bytes:
+    ...
+
+class Store:
+    ...
+
+class Function:
+    def __init__(self, store: Store, func: Callable[[Any], Any]) -> None:
+        ...
+
+class Module:
+    def __init__(self, store: Store, wasm: bytes) -> None:
+        ...
+
+class Uint8Array:
+    def __getitem__(self, index: Union[int, slice]) -> int:
+        ...
+
+    def __setitem__(self, idx: int, value: int) -> None:
+        ...
+
+class Memory:
+    def uint8_view(self, offset: int = 0) -> Uint8Array:
+        ...
+
+class Exports:
+    ...
+
+class ImportObject:
+    def register(self, region: str, values: Dict[str, Function]) -> None:
+        ...
+
+class Instance:
+    exports: Exports
+
+    def __init__(self, module: Module, imports: ImportObject) -> None:
+        ...

tests/integration/helpers.py

@@ -18,6 +18,8 @@ from phasm.codestyle import phasm_render
 from phasm.compiler import phasm_compile
 from phasm.parser import phasm_parse
 
+from . import runners
+
 DASHES = '-' * 16
 
 def wat2wasm(code_wat):
@@ -44,19 +46,13 @@ def wat2wasm(code_wat):
 
 class SuiteResult:
     def __init__(self):
-        self.log_int32_list = []
         self.returned_value = None
 
-    def callback_log_int32(self, store, value):
+RUNNER_CLASS_MAP = {
-        del store # auto passed by pywasm
+    'pywasm': runners.RunnerPywasm,
-
+    'pywasm3': runners.RunnerPywasm3,
-        self.log_int32_list.append(value)
+    'wasmtime': runners.RunnerWasmtime,
-
+    'wasmer': runners.RunnerWasmer,
-    def make_imports(self):
-        return {
-            'console': {
-                'logInt32': self.callback_log_int32,
-            }
 }
 
 class Suite:
@@ -73,189 +69,60 @@ class Suite:
 
         Returned is an object with the results set
         """
-        phasm_module = phasm_parse(self.code_py)
+        class_ = RUNNER_CLASS_MAP[runtime]
+
+        runner = class_(self.code_py)
+
+        runner.parse()
+        runner.compile_ast()
+        runner.compile_wat()
+        runner.compile_wasm()
+        runner.interpreter_setup()
+        runner.interpreter_load(imports)
+
+        write_header(sys.stderr, 'Phasm')
+        runner.dump_phasm_code(sys.stderr)
+        write_header(sys.stderr, 'Assembly')
+        runner.dump_wasm_wat(sys.stderr)
 
         # Check if code formatting works
-        assert self.code_py == '\n' + phasm_render(phasm_module) # \n for formatting in tests
+        assert self.code_py == '\n' + phasm_render(runner.phasm_ast) # \n for formatting in tests
 
-        # Compile
+        # runner.call('stdlib.alloc.__init__')
-        wasm_module = phasm_compile(phasm_module)
 
-        # Render as WebAssembly text
+        wasm_args = []
-        code_wat = wasm_module.to_wat()
+        if args:
+            write_header(sys.stderr, 'Memory (pre alloc)')
+            runner.interpreter_dump_memory(sys.stderr)
 
-        sys.stderr.write(f'{DASHES} Assembly {DASHES}\n')
-
-        _write_numbered_lines(code_wat)
-
-        # Compile to assembly code
-        code_wasm = wat2wasm(code_wat)
-
-        # Run assembly code
-        if 'pywasm' == runtime:
-            return _run_pywasm(code_wasm, args)
-        if 'pywasm3' == runtime:
-            return _run_pywasm3(code_wasm, args)
-        if 'wasmtime' == runtime:
-            return _run_wasmtime(code_wasm, args)
-        if 'wasmer' == runtime:
-            return _run_wasmer(code_wasm, args, imports)
-
-        raise Exception(f'Invalid runtime: {runtime}')
-
-def _run_pywasm(code_wasm, args):
-    # https://pypi.org/project/pywasm/
-    result = SuiteResult()
-
-    module = pywasm.binary.Module.from_reader(io.BytesIO(code_wasm))
-
-    runtime = pywasm.Runtime(module, result.make_imports(), {})
-
-    def set_byte(idx, byt):
-        runtime.store.mems[0].data[idx] = byt
-
-    args = _convert_bytes_arguments(
-        args,
-        lambda x: runtime.exec('___new_reference___', [x]),
-        set_byte
-    )
-
-    sys.stderr.write(f'{DASHES} Memory (pre run)  {DASHES}\n')
-    _dump_memory(runtime.store.mems[0].data)
-
-    result.returned_value = runtime.exec('testEntry', args)
-
-    sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n')
-    _dump_memory(runtime.store.mems[0].data)
-
-    return result
-
-def _run_pywasm3(code_wasm, args):
-    # https://pypi.org/project/pywasm3/
-    result = SuiteResult()
-
-    env = wasm3.Environment()
-
-    mod = env.parse_module(code_wasm)
-
-    rtime = env.new_runtime(1024 * 1024)
-    rtime.load(mod)
-
-    def set_byte(idx, byt):
-        rtime.get_memory(0)[idx] = byt
-
-    args = _convert_bytes_arguments(
-        args,
-        rtime.find_function('___new_reference___'),
-        set_byte
-    )
-
-    sys.stderr.write(f'{DASHES} Memory (pre run)  {DASHES}\n')
-    _dump_memory(rtime.get_memory(0))
-
-    result.returned_value = rtime.find_function('testEntry')(*args)
-
-    sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n')
-    _dump_memory(rtime.get_memory(0))
-
-    return result
-
-def _run_wasmtime(code_wasm, args):
-    # https://pypi.org/project/wasmtime/
-    result = SuiteResult()
-
-    store = wasmtime.Store()
-
-    module = wasmtime.Module(store.engine, code_wasm)
-
-    instance = wasmtime.Instance(store, module, [])
-
-    sys.stderr.write(f'{DASHES} Memory (pre run)  {DASHES}\n')
-    sys.stderr.write('<Not available on wasmtime>\n')
-
-    result.returned_value = instance.exports(store)['testEntry'](store, *args)
-
-    sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n')
-    sys.stderr.write('<Not available on wasmtime>\n')
-
-    return result
-
-def _run_wasmer(code_wasm, args, imports):
-    # https://pypi.org/project/wasmer/
-    result = SuiteResult()
-
-    store = wasmer.Store(wasmer.engine.JIT(wasmer_compiler_cranelift.Compiler))
-
-    import_object = wasmer.ImportObject()
-    import_object.register('imports', {
-        k: wasmer.Function(store, v)
-        for k, v in (imports or {}).items()
-    })
-
-    # Let's compile the module to be able to execute it!
-    module = wasmer.Module(store, code_wasm)
-
-    # Now the module is compiled, we can instantiate it.
-    instance = wasmer.Instance(module, import_object)
-
-    sys.stderr.write(f'{DASHES} Memory (pre run)  {DASHES}\n')
-    sys.stderr.write('<Not available on wasmer>\n')
-
-    # Call the exported `sum` function.
-    result.returned_value = instance.exports.testEntry(*args)
-
-    sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n')
-    sys.stderr.write('<Not available on wasmer>\n')
-
-    return result
-
-def _convert_bytes_arguments(args, new_reference, set_byte):
-    result = []
             for arg in args:
-        if not isinstance(arg, bytes):
+                if isinstance(arg, (int, float, )):
-            result.append(arg)
+                    wasm_args.append(arg)
                     continue
 
+                if isinstance(arg, bytes):
                     # TODO: Implement and use the bytes constructor function
-        offset = new_reference(len(arg) + 4)
+                    # TODO: call upon stdlib.alloc.__init__ and stdlib.alloc.__alloc__
-        result.append(offset)
+                    adr = runner.call('___new_reference___', len(arg) + 4)
+                    sys.stderr.write(f'Allocation 0x{adr:08x} {repr(arg)}\n')
 
-        # Store the length prefix
+                    runner.interpreter_write_memory(adr, len(arg).to_bytes(4, byteorder='little'))
-        for idx, byt in enumerate(len(arg).to_bytes(4, byteorder='little')):
+                    runner.interpreter_write_memory(adr + 4, arg)
-            set_byte(offset + idx, byt)
+                    wasm_args.append(adr)
+                    continue
 
-        # Store the actual bytes
+                raise NotImplementedError(arg)
-        for idx, byt in enumerate(arg):
+
-            set_byte(offset + 4 + idx, byt)
+        write_header(sys.stderr, 'Memory (pre run)')
+        runner.interpreter_dump_memory(sys.stderr)
+
+        result = SuiteResult()
+        result.returned_value = runner.call('testEntry', *wasm_args)
+
+        write_header(sys.stderr, 'Memory (post run)')
+        runner.interpreter_dump_memory(sys.stderr)
 
         return result
 
-def _dump_memory(mem):
+def write_header(textio, msg: str) -> None:
-    line_width = 16
+    textio.write(f'{DASHES} {msg.ljust(16)}  {DASHES}\n')
-
-    prev_line = None
-    skip = False
-    for idx in range(0, len(mem), line_width):
-        line = ''
-        for idx2 in range(0, line_width):
-            line += f'{mem[idx + idx2]:02X}'
-            if idx2 % 2 == 1:
-                line += ' '
-
-        if prev_line == line:
-            if not skip:
-                sys.stderr.write('**\n')
-            skip = True
-        else:
-            sys.stderr.write(f'{idx:08x} {line}\n')
-
-        prev_line = line
-
-def _write_numbered_lines(text: str) -> None:
-    line_list = text.split('\n')
-    line_no_width = len(str(len(line_list)))
-    for line_no, line_txt in enumerate(line_list):
-        sys.stderr.write('{}  {}\n'.format(
-            str(line_no + 1).zfill(line_no_width),
-            line_txt,
-        ))

tests/integration/runners.py

@@ -1,7 +1,7 @@
 """
 Runners to help run WebAssembly code on various interpreters
 """
-from typing import Any, Iterable, TextIO
+from typing import Any, Callable, Dict, Iterable, Optional, TextIO
 
 import ctypes
 import io
@@ -71,7 +71,7 @@ class RunnerBase:
         """
         raise NotImplementedError
 
-    def interpreter_load(self) -> None:
+    def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None:
         """
         Loads the code into the interpreter
         """
@@ -114,7 +114,10 @@ class RunnerPywasm(RunnerBase):
         # Nothing to set up
         pass
 
-    def interpreter_load(self) -> None:
+    def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None:
+        if imports is not None:
+            raise NotImplementedError
+
         bytesio = io.BytesIO(self.wasm_bin)
         self.module = pywasm.binary.Module.from_reader(bytesio)
         self.runtime = pywasm.Runtime(self.module, {}, None)
@@ -146,7 +149,10 @@ class RunnerPywasm3(RunnerBase):
         self.env = wasm3.Environment()
         self.rtime = self.env.new_runtime(1024 * 1024)
 
-    def interpreter_load(self) -> None:
+    def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None:
+        if imports is not None:
+            raise NotImplementedError
+
         self.mod = self.env.parse_module(self.wasm_bin)
         self.rtime.load(self.mod)
 
@@ -179,7 +185,10 @@ class RunnerWasmtime(RunnerBase):
     def interpreter_setup(self) -> None:
         self.store = wasmtime.Store()
 
-    def interpreter_load(self) -> None:
+    def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None:
+        if imports is not None:
+            raise NotImplementedError
+
         self.module = wasmtime.Module(self.store.engine, self.wasm_bin)
         self.instance = wasmtime.Instance(self.store, self.module, [])
 
@@ -242,9 +251,16 @@ class RunnerWasmer(RunnerBase):
     def interpreter_setup(self) -> None:
         self.store = wasmer.Store()
 
-    def interpreter_load(self) -> None:
+    def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None:
+        import_object = wasmer.ImportObject()
+        if imports:
+            import_object.register('imports', {
+                k: wasmer.Function(self.store, v)
+                for k, v in (imports or {}).items()
+            })
+
         self.module = wasmer.Module(self.store, self.wasm_bin)
-        self.instance = wasmer.Instance(self.module)
+        self.instance = wasmer.Instance(self.module, import_object)
 
     def interpreter_write_memory(self, offset: int, data: Iterable[int]) -> None:
         exports = self.instance.exports

tests/integration/test_simple.py

@@ -114,7 +114,6 @@ def testEntry(a: i32) -> i64:
     result = Suite(code_py).run_code(125)
 
     assert 125 == result.returned_value
-    assert [] == result.log_int32_list
 
 @pytest.mark.integration_test
 @pytest.mark.skip('Do we want it to work like this?')
@@ -128,7 +127,6 @@ def testEntry(a: i32, b: i64) -> i64:
     result = Suite(code_py).run_code(125, 100)
 
     assert 225 == result.returned_value
-    assert [] == result.log_int32_list
 
 @pytest.mark.integration_test
 @pytest.mark.skip('Do we want it to work like this?')
@@ -142,7 +140,6 @@ def testEntry(a: f32) -> f64:
     result = Suite(code_py).run_code(125.5)
 
     assert 125.5 == result.returned_value
-    assert [] == result.log_int32_list
 
 @pytest.mark.integration_test
 @pytest.mark.skip('Do we want it to work like this?')
@@ -156,7 +153,6 @@ def testEntry(a: f32, b: f64) -> f64:
     result = Suite(code_py).run_code(125.5, 100.25)
 
     assert 225.75 == result.returned_value
-    assert [] == result.log_int32_list
 
 @pytest.mark.integration_test
 @pytest.mark.skip('TODO')
@@ -170,7 +166,6 @@ def testEntry() -> i32:
     result = Suite(code_py).run_code()
 
     assert 523 == result.returned_value
-    assert [] == result.log_int32_list
 
 @pytest.mark.integration_test
 @pytest.mark.skip('TODO')
@@ -184,7 +179,6 @@ def testEntry() -> i32:
     result = Suite(code_py).run_code()
 
     assert -19 == result.returned_value
-    assert [] == result.log_int32_list
 
 @pytest.mark.integration_test
 @pytest.mark.parametrize('inp', [9, 10, 11, 12])
@@ -268,7 +262,6 @@ def testEntry() -> i32:
     result = Suite(code_py).run_code()
 
     assert 13 == result.returned_value
-    assert [] == result.log_int32_list
 
 @pytest.mark.integration_test
 def test_call_post_defined():
@@ -284,7 +277,6 @@ def helper(left: i32, right: i32) -> i32:
     result = Suite(code_py).run_code()
 
     assert 7 == result.returned_value
-    assert [] == result.log_int32_list
 
 @pytest.mark.integration_test
 @pytest.mark.parametrize('type_', COMPLETE_SIMPLE_TYPES)
@@ -317,7 +309,6 @@ def testEntry() -> i32:
     result = Suite(code_py).run_code()
 
     assert 8947 == result.returned_value
-    assert [] == result.log_int32_list
 
 @pytest.mark.integration_test
 @pytest.mark.parametrize('type_', TYPE_MAP.keys())
@@ -337,7 +328,6 @@ def helper(cv: CheckedValue) -> {type_}:
     result = Suite(code_py).run_code()
 
     assert 23 == result.returned_value
-    assert [] == result.log_int32_list
 
 @pytest.mark.integration_test
 def test_struct_1():
@@ -358,7 +348,6 @@ def helper(shape: Rectangle) -> i32:
     result = Suite(code_py).run_code()
 
     assert 252 == result.returned_value
-    assert [] == result.log_int32_list
 
 @pytest.mark.integration_test
 def test_struct_2():
@@ -379,7 +368,6 @@ def helper(shape1: Rectangle, shape2: Rectangle) -> i32:
     result = Suite(code_py).run_code()
 
     assert 545 == result.returned_value
-    assert [] == result.log_int32_list
 
 @pytest.mark.integration_test
 @pytest.mark.parametrize('type_', COMPLETE_SIMPLE_TYPES)
@@ -412,7 +400,6 @@ def helper(v: (f32, f32, f32, )) -> f32:
     result = Suite(code_py).run_code()
 
     assert 3.74 < result.returned_value < 3.75
-    assert [] == result.log_int32_list
 
 @pytest.mark.integration_test
 def test_bytes_address():

tests/integration/test_stdlib_alloc.py

@@ -2,7 +2,7 @@ import sys
 
 import pytest
 
-from .helpers import DASHES
+from .helpers import write_header
 from .runners import RunnerPywasm3 as Runner
 
 def setup_interpreter(phash_code: str) -> Runner:
@@ -15,9 +15,9 @@ def setup_interpreter(phash_code: str) -> Runner:
     runner.interpreter_setup()
     runner.interpreter_load()
 
-    sys.stderr.write(f'{DASHES} Phasm {DASHES}\n')
+    write_header(sys.stderr, 'Phasm')
     runner.dump_phasm_code(sys.stderr)
-    sys.stderr.write(f'{DASHES} Assembly {DASHES}\n')
+    write_header(sys.stderr, 'Assembly')
     runner.dump_wasm_wat(sys.stderr)
 
     return runner
@@ -35,12 +35,12 @@ def testEntry() -> u8:
     # Garbage in the memory so we can test for it
     runner.interpreter_write_memory(0, range(128))
 
-    sys.stderr.write(f'{DASHES} Memory (pre run)  {DASHES}\n')
+    write_header(sys.stderr, 'Memory (pre run)')
     runner.interpreter_dump_memory(sys.stderr)
 
     runner.call('stdlib.alloc.__init__')
 
-    sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n')
+    write_header(sys.stderr, 'Memory (post run)')
     runner.interpreter_dump_memory(sys.stderr)
 
     assert (
@@ -61,7 +61,7 @@ def testEntry() -> u8:
 
     runner = setup_interpreter(code_py)
 
-    sys.stderr.write(f'{DASHES} Memory (pre run)  {DASHES}\n')
+    write_header(sys.stderr, 'Memory (pre run)')
     runner.interpreter_dump_memory(sys.stderr)
 
     with pytest.raises(Exception, match='unreachable'):
@@ -77,7 +77,7 @@ def testEntry() -> u8:
 
     runner = setup_interpreter(code_py)
 
-    sys.stderr.write(f'{DASHES} Memory (pre run)  {DASHES}\n')
+    write_header(sys.stderr, 'Memory (pre run)')
     runner.interpreter_dump_memory(sys.stderr)
 
     runner.call('stdlib.alloc.__init__')
@@ -85,7 +85,7 @@ def testEntry() -> u8:
     offset1 = runner.call('stdlib.alloc.__alloc__', 32)
     offset2 = runner.call('stdlib.alloc.__alloc__', 32)
 
-    sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n')
+    write_header(sys.stderr, 'Memory (post run)')
     runner.interpreter_dump_memory(sys.stderr)
 
     assert (