From 5c537f712e8e46743204ab9d8d2ff13d8b6e295a Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 5 Apr 2025 15:43:49 +0200 Subject: [PATCH] Project update Various updates to bring the project uptodate. - Updated required packages - Removed runtimes that are not being updated - wasmtime is for now the only supported runtime - Implements imports for wasmtime runtime - Fixes a memory access bug for wasmtime runtime - compile_wasm is now optional - runtimes have to implement and call this themselves - Typing fixes - Linting fixes --- Makefile | 2 +- phasm/parser.py | 2 +- requirements.txt | 27 ++- tests/integration/helpers.py | 6 +- tests/integration/runners.py | 174 ++++-------------- tests/integration/test_examples/test_crc32.py | 1 + tests/integration/test_lang/generator.py | 3 +- tests/integration/test_lang/test_builtins.py | 15 +- tests/integration/test_lang/test_imports.py | 3 - tests/integration/test_lang/test_num.py | 12 +- .../test_lang/test_static_array.py | 3 +- tests/integration/test_stdlib/test_alloc.py | 3 +- 12 files changed, 75 insertions(+), 176 deletions(-) diff --git a/Makefile b/Makefile index dc4a68b..c0b5d5b 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ typecheck: venv/.done venv/bin/mypy --strict phasm tests/integration/helpers.py tests/integration/runners.py venv/.done: requirements.txt - python3.10 -m venv venv + python3.12 -m venv venv venv/bin/python3 -m pip install wheel pip --upgrade venv/bin/python3 -m pip install -r $^ touch $@ diff --git a/phasm/parser.py b/phasm/parser.py index 0aaa2b6..15f4992 100644 --- a/phasm/parser.py +++ b/phasm/parser.py @@ -675,7 +675,7 @@ def _not_implemented(check: Any, msg: str) -> None: if not check: raise NotImplementedError(msg) -def _raise_static_error(node: Union[ast.mod, ast.stmt, ast.expr], msg: str) -> NoReturn: +def _raise_static_error(node: Union[ast.stmt, ast.expr], msg: str) -> NoReturn: raise StaticError( f'Static error on line {node.lineno}: {msg}' ) diff --git a/requirements.txt b/requirements.txt index f797665..c52ffeb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,19 @@ -mypy==0.991 -pygments==2.12.0 -pytest==7.2.0 +marko==2.1.3 +mypy==1.15.0 +pygments==2.19.1 +pytest==8.3.5 pytest-integration==0.2.2 -pywasm==1.0.7 -pywasm3==0.5.0 -ruff==0.1.5 -wasmer==1.1.0 -wasmer_compiler_cranelift==1.1.0 -wasmtime==3.0.0 +ruff==0.11.4 + + +wasmtime==31.0.0 + +# TODO: +# extism? +# wasmedge + +# Check 2025-04-05 +# wasm3: minimal maintenance phase +# py-wasm: last updated 6 years ago +# wasmer-python: Not compatible with python3.12, last updated 2 years ago +# WAVM: Last updated 3 years ago diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index db4574f..3ea3aef 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -16,10 +16,7 @@ class SuiteResult: self.returned_value = None RUNNER_CLASS_MAP = { - 'pywasm': runners.RunnerPywasm, - 'pywasm3': runners.RunnerPywasm3, 'wasmtime': runners.RunnerWasmtime, - 'wasmer': runners.RunnerWasmer, } class Suite: @@ -29,7 +26,7 @@ class Suite: def __init__(self, code_py: str) -> None: self.code_py = code_py - def run_code(self, *args: Any, runtime: str = 'pywasm3', func_name: str = 'testEntry', imports: runners.Imports = None) -> Any: + def run_code(self, *args: Any, runtime: str = 'wasmtime', func_name: str = 'testEntry', imports: runners.Imports = None) -> Any: """ Compiles the given python code into wasm and then runs it @@ -50,7 +47,6 @@ class Suite: write_header(sys.stderr, 'Assembly') runner.dump_wasm_wat(sys.stderr) - runner.compile_wasm() runner.interpreter_setup() runner.interpreter_load(imports) diff --git a/tests/integration/runners.py b/tests/integration/runners.py index f608f96..6afeacf 100644 --- a/tests/integration/runners.py +++ b/tests/integration/runners.py @@ -2,12 +2,8 @@ Runners to help run WebAssembly code on various interpreters """ import ctypes -import io from typing import Any, Callable, Dict, Iterable, Optional, TextIO -import pywasm.binary -import wasm3 -import wasmer import wasmtime from phasm import ourlang, wasm @@ -65,7 +61,7 @@ class RunnerBase: """ Compiles the WebAssembly AST into WebAssembly Binary """ - self.wasm_bin = wasmer.wat2wasm(self.wasm_asm) + raise NotImplementedError def interpreter_setup(self) -> None: """ @@ -103,77 +99,6 @@ class RunnerBase: """ raise NotImplementedError -class RunnerPywasm(RunnerBase): - """ - Implements a runner for pywasm - - See https://pypi.org/project/pywasm/ - """ - module: pywasm.binary.Module - runtime: pywasm.Runtime - - def interpreter_setup(self) -> None: - # Nothing to set up - pass - - def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None: - if imports is not None: - raise NotImplementedError - - bytesio = io.BytesIO(self.wasm_bin) - self.module = pywasm.binary.Module.from_reader(bytesio) - self.runtime = pywasm.Runtime(self.module, {}, None) - - def interpreter_write_memory(self, offset: int, data: Iterable[int]) -> None: - for idx, byt in enumerate(data): - self.runtime.store.memory_list[0].data[offset + idx] = byt - - def interpreter_read_memory(self, offset: int, length: int) -> bytes: - return self.runtime.store.memory_list[0].data[offset:length] - - def interpreter_dump_memory(self, textio: TextIO) -> None: - _dump_memory(textio, self.runtime.store.memory_list[0].data) - - def call(self, function: str, *args: Any) -> Any: - return self.runtime.exec(function, [*args]) - -class RunnerPywasm3(RunnerBase): - """ - Implements a runner for pywasm3 - - See https://pypi.org/project/pywasm3/ - """ - env: wasm3.Environment - rtime: wasm3.Runtime - mod: wasm3.Module - - def interpreter_setup(self) -> None: - self.env = wasm3.Environment() - self.rtime = self.env.new_runtime(1024 * 1024) - - def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None: - if imports is not None: - raise NotImplementedError - - self.mod = self.env.parse_module(self.wasm_bin) - self.rtime.load(self.mod) - - def interpreter_write_memory(self, offset: int, data: Iterable[int]) -> None: - memory = self.rtime.get_memory(0) - - for idx, byt in enumerate(data): - memory[offset + idx] = byt - - def interpreter_read_memory(self, offset: int, length: int) -> bytes: - memory = self.rtime.get_memory(0) - return memory[offset:offset + length].tobytes() - - def interpreter_dump_memory(self, textio: TextIO) -> None: - _dump_memory(textio, self.rtime.get_memory(0)) - - def call(self, function: str, *args: Any) -> Any: - return self.rtime.find_function(function)(*args) - class RunnerWasmtime(RunnerBase): """ Implements a runner for wasmtime @@ -184,15 +109,44 @@ class RunnerWasmtime(RunnerBase): module: wasmtime.Module instance: wasmtime.Instance + @classmethod + def func2type(cls, func: Callable[[Any], Any]) -> wasmtime.FuncType: + params: list[wasmtime.ValType] = [] + + code = func.__code__ + for idx in range(code.co_argcount): + varname = code.co_varnames[idx] + vartype = func.__annotations__[varname] + + if vartype is int: + params.append(wasmtime.ValType.i32()) + else: + raise NotImplementedError + + results: list[wasmtime.ValType] = [] + if func.__annotations__['return'] is None: + pass # No return value + elif func.__annotations__['return'] is int: + results.append(wasmtime.ValType.i32()) + else: + raise NotImplementedError('Return type', func.__annotations__['return']) + + return wasmtime.FuncType(params, results) + def interpreter_setup(self) -> None: self.store = wasmtime.Store() def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None: - if imports is not None: - raise NotImplementedError + functions: list[wasmtime.Func] = [] - self.module = wasmtime.Module(self.store.engine, self.wasm_bin) - self.instance = wasmtime.Instance(self.store, self.module, []) + if imports is not None: + functions = [ + wasmtime.Func(self.store, self.__class__.func2type(f), f) + for f in imports.values() + ] + + self.module = wasmtime.Module(self.store.engine, self.wasm_asm) + self.instance = wasmtime.Instance(self.store, self.module, functions) def interpreter_write_memory(self, offset: int, data: Iterable[int]) -> None: exports = self.instance.exports(self.store) @@ -217,8 +171,7 @@ class RunnerWasmtime(RunnerBase): data_len = memory.data_len(self.store) raw = ctypes.string_at(data_ptr, data_len) - - return raw[offset:length] + return raw[offset:offset + length] def interpreter_dump_memory(self, textio: TextIO) -> None: exports = self.instance.exports(self.store) @@ -237,63 +190,6 @@ class RunnerWasmtime(RunnerBase): return func(self.store, *args) -class RunnerWasmer(RunnerBase): - """ - Implements a runner for wasmer - - See https://pypi.org/project/wasmer/ - """ - - # pylint: disable=E1101 - - store: wasmer.Store - module: wasmer.Module - instance: wasmer.Instance - - def interpreter_setup(self) -> None: - self.store = wasmer.Store() - - def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None: - import_object = wasmer.ImportObject() - if imports: - import_object.register('imports', { - k: wasmer.Function(self.store, v) - for k, v in (imports or {}).items() - }) - - self.module = wasmer.Module(self.store, self.wasm_bin) - self.instance = wasmer.Instance(self.module, import_object) - - def interpreter_write_memory(self, offset: int, data: Iterable[int]) -> None: - exports = self.instance.exports - memory = getattr(exports, 'memory') - assert isinstance(memory, wasmer.Memory) - view = memory.uint8_view(offset) - - for idx, byt in enumerate(data): - view[idx] = byt - - def interpreter_read_memory(self, offset: int, length: int) -> bytes: - exports = self.instance.exports - memory = getattr(exports, 'memory') - assert isinstance(memory, wasmer.Memory) - view = memory.uint8_view(offset) - return bytes(view[offset:length]) - - def interpreter_dump_memory(self, textio: TextIO) -> None: - exports = self.instance.exports - memory = getattr(exports, 'memory') - assert isinstance(memory, wasmer.Memory) - view = memory.uint8_view() - - _dump_memory(textio, view) # type: ignore - - def call(self, function: str, *args: Any) -> Any: - exports = self.instance.exports - func = getattr(exports, function) - - return func(*args) - def _dump_memory(textio: TextIO, mem: bytes) -> None: line_width = 16 diff --git a/tests/integration/test_examples/test_crc32.py b/tests/integration/test_examples/test_crc32.py index f768d6f..976e7d4 100644 --- a/tests/integration/test_examples/test_crc32.py +++ b/tests/integration/test_examples/test_crc32.py @@ -24,6 +24,7 @@ def test_crc32(): def _crc32_f(crc: u32, byt: u8) -> u32: return 16777215 ^ 397917763 +@exported def testEntry(data: bytes) -> u32: return 4294967295 ^ _crc32_f(4294967295, data[0]) """ diff --git a/tests/integration/test_lang/generator.py b/tests/integration/test_lang/generator.py index 771f363..045f8c3 100644 --- a/tests/integration/test_lang/generator.py +++ b/tests/integration/test_lang/generator.py @@ -102,6 +102,7 @@ def generate_code(markdown, template, settings): print() print('from ..helpers import Suite') print() + print() for test in get_tests(template): assert len(test) == 4, test @@ -121,7 +122,7 @@ def generate_code(markdown, template, settings): print('@pytest.mark.integration_test') print(f'def test_{type_name}_{test_id}():') print(' """') - print(' ' + user_story.replace('\n', '\n ')) + print(' ' + user_story.strip().replace('\n', '\n ')) print(' """') print(' code_py = """') if 'CODE_HEADER' in settings: diff --git a/tests/integration/test_lang/test_builtins.py b/tests/integration/test_lang/test_builtins.py index 756e679..22ed10c 100644 --- a/tests/integration/test_lang/test_builtins.py +++ b/tests/integration/test_lang/test_builtins.py @@ -3,16 +3,15 @@ import sys import pytest from ..helpers import Suite, write_header -from ..runners import RunnerPywasm +from ..runners import RunnerWasmtime -def setup_interpreter(phash_code: str) -> RunnerPywasm: - runner = RunnerPywasm(phash_code) +def setup_interpreter(phash_code: str) -> RunnerWasmtime: + runner = RunnerWasmtime(phash_code) runner.parse() runner.compile_ast() runner.compile_wat() - runner.compile_wasm() runner.interpreter_setup() runner.interpreter_load() @@ -38,16 +37,16 @@ def testEntry(b: bytes) -> u8: result = suite.run_code(b'') assert 128 == result.returned_value - result = suite.run_code(b'\x80', runtime='pywasm') + result = suite.run_code(b'\x80') assert 128 == result.returned_value - result = suite.run_code(b'\x80\x40', runtime='pywasm') + result = suite.run_code(b'\x80\x40') assert 192 == result.returned_value - result = suite.run_code(b'\x80\x40\x20\x10', runtime='pywasm') + result = suite.run_code(b'\x80\x40\x20\x10') assert 240 == result.returned_value - result = suite.run_code(b'\x80\x40\x20\x10\x08\x04\x02\x01', runtime='pywasm') + result = suite.run_code(b'\x80\x40\x20\x10\x08\x04\x02\x01') assert 255 == result.returned_value @pytest.mark.integration_test diff --git a/tests/integration/test_lang/test_imports.py b/tests/integration/test_lang/test_imports.py index 1762a0b..9c0b93d 100644 --- a/tests/integration/test_lang/test_imports.py +++ b/tests/integration/test_lang/test_imports.py @@ -21,7 +21,6 @@ def testEntry() -> i32: return 4238 * mul result = Suite(code_py).run_code( - runtime='wasmer', imports={ 'helper': helper, } @@ -47,7 +46,6 @@ def testEntry() -> None: prop = mul result = Suite(code_py).run_code( - runtime='wasmer', imports={ 'helper': helper, } @@ -73,7 +71,6 @@ def testEntry(x: u32) -> u8: with pytest.raises(Type3Exception, match=r'u32 must be u8 instead'): Suite(code_py).run_code( - runtime='wasmer', imports={ 'helper': helper, } diff --git a/tests/integration/test_lang/test_num.py b/tests/integration/test_lang/test_num.py index d2db0cf..7cd0547 100644 --- a/tests/integration/test_lang/test_num.py +++ b/tests/integration/test_lang/test_num.py @@ -26,7 +26,7 @@ def testEntry() -> {type_}: result = Suite(code_py).run_code() assert 13 == result.returned_value - assert TYPE_MAP[type_] == type(result.returned_value) + assert TYPE_MAP[type_] is type(result.returned_value) @pytest.mark.integration_test @pytest.mark.parametrize('type_', FLOAT_TYPES) @@ -40,7 +40,7 @@ def testEntry() -> {type_}: result = Suite(code_py).run_code() assert 32.125 == result.returned_value - assert TYPE_MAP[type_] == type(result.returned_value) + assert TYPE_MAP[type_] is type(result.returned_value) @pytest.mark.integration_test @pytest.mark.parametrize('type_', INT_TYPES) @@ -54,7 +54,7 @@ def testEntry() -> {type_}: result = Suite(code_py).run_code() assert 7 == result.returned_value - assert TYPE_MAP[type_] == type(result.returned_value) + assert TYPE_MAP[type_] is type(result.returned_value) @pytest.mark.integration_test @pytest.mark.parametrize('type_', FLOAT_TYPES) @@ -68,7 +68,7 @@ def testEntry() -> {type_}: result = Suite(code_py).run_code() assert 32.125 == result.returned_value - assert TYPE_MAP[type_] == type(result.returned_value) + assert TYPE_MAP[type_] is type(result.returned_value) @pytest.mark.integration_test @pytest.mark.skip('TODO: Runtimes return a signed value, which is difficult to test') @@ -101,7 +101,7 @@ def helper(left: {type_}, right: {type_}) -> {type_}: result = Suite(code_py).run_code() assert 22 == result.returned_value - assert TYPE_MAP[type_] == type(result.returned_value) + assert TYPE_MAP[type_] is type(result.returned_value) @pytest.mark.integration_test @pytest.mark.parametrize('type_', FLOAT_TYPES) @@ -118,4 +118,4 @@ def helper(left: {type_}, right: {type_}) -> {type_}: result = Suite(code_py).run_code() assert 32.125 == result.returned_value - assert TYPE_MAP[type_] == type(result.returned_value) + assert TYPE_MAP[type_] is type(result.returned_value) diff --git a/tests/integration/test_lang/test_static_array.py b/tests/integration/test_lang/test_static_array.py index e804363..fa3ed3e 100644 --- a/tests/integration/test_lang/test_static_array.py +++ b/tests/integration/test_lang/test_static_array.py @@ -1,4 +1,5 @@ import pytest +import wasmtime from phasm.type3.entry import Type3Exception @@ -64,7 +65,7 @@ def testEntry(x: u32) -> u8: return CONSTANT[x] """ - with pytest.raises(RuntimeError): + with pytest.raises(wasmtime.Trap): Suite(code_py).run_code(3) @pytest.mark.integration_test diff --git a/tests/integration/test_stdlib/test_alloc.py b/tests/integration/test_stdlib/test_alloc.py index 34a833e..d9f1a60 100644 --- a/tests/integration/test_stdlib/test_alloc.py +++ b/tests/integration/test_stdlib/test_alloc.py @@ -3,7 +3,7 @@ import sys import pytest from ..helpers import write_header -from ..runners import RunnerPywasm3 as Runner +from ..runners import RunnerWasmtime as Runner def setup_interpreter(phash_code: str) -> Runner: @@ -12,7 +12,6 @@ def setup_interpreter(phash_code: str) -> Runner: runner.parse() runner.compile_ast() runner.compile_wat() - runner.compile_wasm() runner.interpreter_setup() runner.interpreter_load()