Improve test information on errors during parsing

phasm/parser.py

@@ -39,11 +39,39 @@ def phasm_parse(source: str) -> Module:
     """
     res = ast.parse(source, '')
 
+    res = OptimizerTransformer().visit(res)
+
     our_visitor = OurVisitor()
     return our_visitor.visit_Module(res)
 
 OurLocals = Dict[str, Union[FunctionParam]] # FIXME: Does it become easier if we add ModuleConstantDef to this dict?
 
+class OptimizerTransformer(ast.NodeTransformer):
+    """
+    This class optimizes the Python AST, to prepare it for parsing
+    by the OurVisitor class below.
+    """
+    def visit_UnaryOp(self, node: ast.UnaryOp) -> Union[ast.UnaryOp, ast.Constant]:
+        """
+        UnaryOp optimizations
+
+        In the given example:
+        ```py
+        x = -4
+        ```
+        Python will parse it as a unary minus operation on the constant four.
+        For Phasm purposes, this counts as a literal -4.
+        """
+        if (
+                isinstance(node.op, (ast.UAdd, ast.USub, ))
+                and isinstance(node.operand, ast.Constant)
+                and isinstance(node.operand.value, (int, float, ))
+            ):
+            if isinstance(node.op, ast.USub):
+                node.operand.value = -node.operand.value
+            return node.operand
+        return node
+
 class OurVisitor:
     """
     Class to visit a Python syntax tree and create an ourlang syntax tree
@@ -52,6 +80,9 @@ class OurVisitor:
 
     At some point, we may deviate from Python syntax. If nothing else,
     we probably won't keep up with the Python syntax changes.
+
+    See OptimizerTransformer for the changes we make after the Python
+    parsing is done but before the phasm parsing is done.
     """
 
     # pylint: disable=C0103,C0116,C0301,R0201,R0912

phasm/wasmgenerator.py

@@ -45,7 +45,7 @@ class Generator_i32i64:
         self.store = functools.partial(self.generator.add_statement, f'{prefix}.store')
 
     def const(self, value: int, comment: Optional[str] = None) -> None:
-        self.generator.add_statement(f'{self.prefix}.const', f'0x{value:08x}', comment=comment)
+        self.generator.add_statement(f'{self.prefix}.const', f'{value}', comment=comment)
 
 class Generator_i32(Generator_i32i64):
     def __init__(self, generator: 'Generator') -> None:

tests/integration/helpers.py

@@ -1,8 +1,9 @@
-from typing import Any, Generator, Iterable, TextIO
+from typing import Any, Generator, Iterable, List, TextIO, Union
 
 import struct
 import sys
 
+from phasm import compiler
 from phasm.codestyle import phasm_render
 from phasm.type3 import types as type3types
 from phasm.runtime import calculate_alloc_size
@@ -40,39 +41,67 @@ class Suite:
 
         runner = class_(self.code_py)
 
+        write_header(sys.stderr, 'Phasm')
+        runner.dump_phasm_code(sys.stderr)
+
         runner.parse()
         runner.compile_ast()
         runner.compile_wat()
+
+        write_header(sys.stderr, 'Assembly')
+        runner.dump_wasm_wat(sys.stderr)
+
         runner.compile_wasm()
         runner.interpreter_setup()
         runner.interpreter_load(imports)
 
-        write_header(sys.stderr, 'Phasm')
-        runner.dump_phasm_code(sys.stderr)
-        write_header(sys.stderr, 'Assembly')
-        runner.dump_wasm_wat(sys.stderr)
-
         # Check if code formatting works
         assert self.code_py == '\n' + phasm_render(runner.phasm_ast) # \n for formatting in tests
 
-        wasm_args = []
+        func_args = [x.type3 for x in runner.phasm_ast.functions[func_name].posonlyargs]
+        if len(func_args) != len(args):
+            raise RuntimeError(f'Invalid number of args for {func_name}')
+
+        wasm_args: List[Union[float, int]] = []
         if args:
             write_header(sys.stderr, 'Memory (pre alloc)')
             runner.interpreter_dump_memory(sys.stderr)
 
-            for arg in args:
+            for arg, arg_typ in zip(args, func_args):
-                if isinstance(arg, (int, float, )):
+                assert not isinstance(arg_typ, type3types.PlaceholderForType), \
+                    'Cannot call polymorphic function from outside'
+
+                if arg_typ in (type3types.u8, type3types.u32, type3types.u64, ):
+                    assert isinstance(arg, int)
                     wasm_args.append(arg)
                     continue
 
-                if isinstance(arg, bytes):
+                if arg_typ in (type3types.i8, type3types.i32, type3types.i64, ):
-                    adr = runner.call('stdlib.types.__alloc_bytes__', len(arg))
+                    assert isinstance(arg, int)
-                    sys.stderr.write(f'Allocation 0x{adr:08x} {repr(arg)}\n')
+                    wasm_args.append(arg)
+                    continue
 
-                    runner.interpreter_write_memory(adr + 4, arg)
+                if arg_typ in (type3types.f32, type3types.f64, ):
+                    assert isinstance(arg, float)
+                    wasm_args.append(arg)
+                    continue
+
+                if arg_typ is type3types.bytes:
+                    adr = _allocate_memory_stored_value(runner, arg_typ, arg)
                     wasm_args.append(adr)
                     continue
 
+                if isinstance(arg_typ, type3types.AppliedType3):
+                    if arg_typ.base is type3types.static_array:
+                        adr = _allocate_memory_stored_value(runner, arg_typ, arg)
+                        wasm_args.append(adr)
+                        continue
+
+                    if arg_typ.base is type3types.tuple:
+                        adr = _allocate_memory_stored_value(runner, arg_typ, arg)
+                        wasm_args.append(adr)
+                        continue
+
                 raise NotImplementedError(arg)
 
         write_header(sys.stderr, 'Memory (pre run)')
@@ -95,6 +124,98 @@ class Suite:
 def write_header(textio: TextIO, msg: str) -> None:
     textio.write(f'{DASHES} {msg.ljust(16)}  {DASHES}\n')
 
+WRITE_LOOKUP_MAP = {
+    'u8': compiler.module_data_u8,
+    'u32': compiler.module_data_u32,
+    'u64': compiler.module_data_u64,
+    'i8': compiler.module_data_i8,
+    'i32': compiler.module_data_i32,
+    'i64': compiler.module_data_i64,
+    'f32': compiler.module_data_f32,
+    'f64': compiler.module_data_f64,
+}
+
+def _write_memory_stored_value(
+        runner: runners.RunnerBase,
+        adr: int,
+        val_typ: type3types.Type3,
+        val: Any,
+    ) -> int:
+    if val_typ is type3types.bytes:
+        adr2 = _allocate_memory_stored_value(runner, val_typ, val)
+        runner.interpreter_write_memory(adr, compiler.module_data_u32(adr2))
+        return 4
+
+    if isinstance(val_typ, type3types.PrimitiveType3):
+        to_write = WRITE_LOOKUP_MAP[val_typ.name](val)
+        runner.interpreter_write_memory(adr, to_write)
+        return len(to_write)
+
+    if isinstance(val_typ, type3types.AppliedType3):
+        if val_typ.base in (type3types.static_array, type3types.tuple, ):
+            adr2 = _allocate_memory_stored_value(runner, val_typ, val)
+            runner.interpreter_write_memory(adr, compiler.module_data_u32(adr2))
+            return 4
+
+    raise NotImplementedError(val_typ, val)
+
+def _allocate_memory_stored_value(
+        runner: runners.RunnerBase,
+        val_typ: type3types.Type3,
+        val: Any
+    ) -> int:
+    if val_typ is type3types.bytes:
+        assert isinstance(val, bytes)
+
+        adr = runner.call('stdlib.types.__alloc_bytes__', len(val))
+        assert isinstance(adr, int)
+
+        sys.stderr.write(f'Allocation 0x{adr:08x} {repr(val)}\n')
+        runner.interpreter_write_memory(adr + 4, val)
+        return adr
+
+    if isinstance(val_typ, type3types.AppliedType3):
+        if val_typ.base is type3types.static_array:
+            assert isinstance(val, tuple)
+
+            alloc_size = calculate_alloc_size(val_typ)
+            adr = runner.call('stdlib.alloc.__alloc__', alloc_size)
+            assert isinstance(adr, int)
+            sys.stderr.write(f'Allocation 0x{adr:08x} {repr(val)}\n')
+
+            val_el_typ = val_typ.args[0]
+            assert not isinstance(val_el_typ, type3types.PlaceholderForType)
+            val_el_alloc_size = calculate_alloc_size(val_el_typ)
+
+            tuple_len_obj = val_typ.args[1]
+            assert isinstance(tuple_len_obj, type3types.IntType3)
+            tuple_len = tuple_len_obj.value
+            assert tuple_len == len(val)
+
+            offset = adr
+            for val_el_val in val:
+                offset += _write_memory_stored_value(runner, offset, val_el_typ, val_el_val)
+            return adr
+
+        if val_typ.base is type3types.tuple:
+            assert isinstance(val, tuple)
+
+            alloc_size = calculate_alloc_size(val_typ)
+            adr = runner.call('stdlib.alloc.__alloc__', alloc_size)
+            assert isinstance(adr, int)
+            sys.stderr.write(f'Allocation 0x{adr:08x} {repr(val)}\n')
+
+            assert len(val) == len(val_typ.args)
+
+            offset = adr
+            for val_el_val, val_el_typ in zip(val, val_typ.args):
+                assert not isinstance(val_el_typ, type3types.PlaceholderForType)
+
+                offset += _write_memory_stored_value(runner, offset, val_el_typ, val_el_val)
+            return adr
+
+    raise NotImplementedError(val_typ, val)
+
 def _load_memory_stored_returned_value(
         runner: runners.RunnerBase,
         func_name: str,
@@ -111,7 +232,19 @@ def _load_memory_stored_returned_value(
 
     if ret_type3 in (type3types.u8, type3types.u32, type3types.u64):
         assert isinstance(wasm_value, int), wasm_value
-        assert 0 <= wasm_value, 'TODO: Extract negative values'
+
+        if wasm_value < 0:
+            # WASM does not support unsigned values through its interface
+            # Cast and then reinterpret
+
+            letter = {
+                'u32': 'i',
+                'u64': 'q',
+            }[ret_type3.name]
+
+            data = struct.pack(f'<{letter}', wasm_value)
+            wasm_value, = struct.unpack(f'<{letter.upper()}', data)
+
         return wasm_value
 
     if ret_type3 in (type3types.f32, type3types.f64, ):
@@ -192,12 +325,11 @@ def _unpack(runner: runners.RunnerBase, typ: type3types.Type3, inp: bytes) -> An
     raise NotImplementedError(typ, inp)
 
 def _load_bytes_from_address(runner: runners.RunnerBase, typ: type3types.Type3, adr: int) -> bytes:
-    sys.stderr.write(f'Reading 0x{adr:08x}\n')
+    sys.stderr.write(f'Reading 0x{adr:08x} {typ:s}\n')
     read_bytes = runner.interpreter_read_memory(adr, 4)
     bytes_len,  = struct.unpack('<I', read_bytes)
 
     adr += 4
-    sys.stderr.write(f'Reading 0x{adr:08x}\n')
     return runner.interpreter_read_memory(adr, bytes_len)
 
 def _split_read_bytes(all_bytes: bytes, split_sizes: Iterable[int]) -> Generator[bytes, None, None]:
@@ -207,6 +339,8 @@ def _split_read_bytes(all_bytes: bytes, split_sizes: Iterable[int]) -> Generator
         offset += size
 
 def _load_static_array_from_address(runner: runners.RunnerBase, typ: type3types.AppliedType3, adr: int) -> Any:
+    sys.stderr.write(f'Reading 0x{adr:08x} {typ:s}\n')
+
     assert 2 == len(typ.args)
     sub_typ, len_typ = typ.args
 
@@ -226,6 +360,8 @@ def _load_static_array_from_address(runner: runners.RunnerBase, typ: type3types.
     )
 
 def _load_tuple_from_address(runner: runners.RunnerBase, typ: type3types.Type3, adr: int) -> Any:
+    sys.stderr.write(f'Reading 0x{adr:08x} {typ:s}\n')
+
     assert isinstance(typ, type3types.AppliedType3)
     assert typ.base is type3types.tuple
 

tests/integration/test_examples/test_crc32.py

@@ -31,9 +31,4 @@ def testEntry(data: bytes) -> u32:
 
     result = Suite(code_py).run_code(b'a')
 
-    # exp_result returns a unsigned integer, as is proper
+    assert exp_result == result.returned_value
-    exp_data = struct.pack('I', exp_result)
-    # ints extracted from WebAssembly are always signed
-    data = struct.pack('i', result.returned_value)
-
-    assert exp_data == data

tests/integration/test_lang/generator.md

@@ -1,4 +1,4 @@
-# runtime_extract_value
+# runtime_extract_value_literal
 
 As a developer
 I want to extract a $TYPE value
@@ -14,6 +14,22 @@ def testEntry() -> $TYPE:
 expect(VAL0)
 ```
 
+# runtime_extract_value_round_trip
+
+As a developer
+I want to extract a $TYPE value that I've input
+In order let the code select a value that I've predefined
+
+```py
+@exported
+def testEntry(x: $TYPE) -> $TYPE:
+    return x
+```
+
+```py
+expect(VAL0, given=[VAL0])
+```
+
 # module_constant_def_ok
 
 As a developer

tests/integration/test_lang/generator.py

@@ -30,8 +30,10 @@ def apply_settings(settings, txt):
 		txt = txt.replace(f'${k}', v)
 	return txt
 
-def generate_assertion_expect(result, arg):
+def generate_assertion_expect(result, arg, given=None):
-	result.append('result = Suite(code_py).run_code()')
+	given = given or []
+
+	result.append('result = Suite(code_py).run_code(' + ', '.join(repr(x) for x in given) + ')')
 	result.append(f'assert {repr(arg)} == result.returned_value')
 
 def generate_assertion_expect_type_error(result, error_msg, error_comment = None):

tests/integration/test_lang/generator_tuple_all_primitives.json

@@ -1,5 +1,5 @@
 {
 	"TYPE_NAME": "tuple_all_primitives",
-	"TYPE": "(u8, u32, u64, i8, i32, i64, f32, f64, bytes, )",
+	"TYPE": "(u8, u32, u64, i8, i8, i32, i32, i64, i64, f32, f32, f64, f64, bytes, )",
-	"VAL0": "(1, 4, 8, 1, 4, 8, 125.125, 5000.5, b'Hello, world!', )"
+	"VAL0": "(1, 4, 8, 1, -1, 4, -4, 8, -8, 125.125, -125.125, 5000.5, -5000.5, b'Hello, world!', )"
 }

tests/integration/test_lang/test_bytes.py

@@ -4,20 +4,6 @@ from phasm.type3.entry import Type3Exception
 
 from ..helpers import Suite
 
-@pytest.mark.integration_test
-def test_bytes_address():
-    code_py = """
-@exported
-def testEntry(f: bytes) -> bytes:
-    return f
-"""
-
-    result = Suite(code_py).run_code(b'This is a test')
-
-    # THIS DEPENDS ON THE ALLOCATOR
-    # A different allocator will return a different value
-    assert 20 == result.returned_value
-
 @pytest.mark.integration_test
 def test_bytes_length():
     code_py = """
@@ -42,25 +28,11 @@ def testEntry(f: bytes) -> u8:
 
     assert 0x61 == result.returned_value
 
-@pytest.mark.integration_test
-def test_constant():
-    code_py = """
-CONSTANT: bytes = b'ABCDEF'
-
-@exported
-def testEntry() -> u8:
-    return CONSTANT[0]
-"""
-
-    result = Suite(code_py).run_code()
-
-    assert 0x41 == result.returned_value
-
 @pytest.mark.integration_test
 def test_bytes_index_out_of_bounds():
     code_py = """
 @exported
-def testEntry(f: bytes) -> u8:
+def testEntry(f: bytes, g: bytes) -> u8:
     return f[50]
 """
 

tests/integration/test_lang/test_primitives.py

@@ -5,34 +5,6 @@ from phasm.type3.entry import Type3Exception
 from ..helpers import Suite
 from ..constants import ALL_INT_TYPES, ALL_FLOAT_TYPES, COMPLETE_INT_TYPES, TYPE_MAP
 
-@pytest.mark.integration_test
-@pytest.mark.parametrize('type_', ALL_INT_TYPES)
-def test_expr_constant_int(type_):
-    code_py = f"""
-@exported
-def testEntry() -> {type_}:
-    return 13
-"""
-
-    result = Suite(code_py).run_code()
-
-    assert 13 == result.returned_value
-    assert TYPE_MAP[type_] == type(result.returned_value)
-
-@pytest.mark.integration_test
-@pytest.mark.parametrize('type_', ALL_FLOAT_TYPES)
-def test_expr_constant_float(type_):
-    code_py = f"""
-@exported
-def testEntry() -> {type_}:
-    return 32.125
-"""
-
-    result = Suite(code_py).run_code()
-
-    assert 32.125 == result.returned_value
-    assert TYPE_MAP[type_] == type(result.returned_value)
-
 @pytest.mark.integration_test
 def test_expr_constant_literal_does_not_fit():
     code_py = """
@@ -295,32 +267,6 @@ def testEntry() -> {type_}:
     assert 5 == result.returned_value
     assert TYPE_MAP[type_] == type(result.returned_value)
 
-@pytest.mark.integration_test
-@pytest.mark.skip('TODO')
-def test_explicit_positive_number():
-    code_py = """
-@exported
-def testEntry() -> i32:
-    return +523
-"""
-
-    result = Suite(code_py).run_code()
-
-    assert 523 == result.returned_value
-
-@pytest.mark.integration_test
-@pytest.mark.skip('TODO')
-def test_explicit_negative_number():
-    code_py = """
-@exported
-def testEntry() -> i32:
-    return -19
-"""
-
-    result = Suite(code_py).run_code()
-
-    assert -19 == result.returned_value
-
 @pytest.mark.integration_test
 def test_call_pre_defined():
     code_py = """