jbwdevries/phasm
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Cleanup to helpers, making use of runners

Browse Source
This commit is contained in:
Johan B.W. de Vries 2022-08-09 19:04:10 +02:00
parent 41b47e43d6
commit a13713d709
6 changed files with 123 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
TODO.md
View File

@ -2,5 +2,5 @@
- Rename ourlang.Struct to OurTypeStruct
- Maybe rename this to a types module?
- Remove references to log_int32_list
- Fix naming in Suite() calls
- Replace ___new_reference___ by stdlib.alloc.__alloc__

39
stubs/wasmer.pyi Normal file
View File

@ -0,0 +1,39 @@
from typing import Any, Dict, Callable, Union
def wat2wasm(inp: str) -> bytes:
...
class Store:
...
class Function:
def __init__(self, store: Store, func: Callable[[Any], Any]) -> None:
...
class Module:
def __init__(self, store: Store, wasm: bytes) -> None:
...
class Uint8Array:
def __getitem__(self, index: Union[int, slice]) -> int:
...
def __setitem__(self, idx: int, value: int) -> None:
...
class Memory:
def uint8_view(self, offset: int = 0) -> Uint8Array:
...
class Exports:
...
class ImportObject:
def register(self, region: str, values: Dict[str, Function]) -> None:
...
class Instance:
exports: Exports
def __init__(self, module: Module, imports: ImportObject) -> None:
...

233
tests/integration/helpers.py
View File

@ -18,6 +18,8 @@ from phasm.codestyle import phasm_render
from phasm.compiler import phasm_compile
from phasm.parser import phasm_parse
from . import runners
DASHES = '-' * 16
def wat2wasm(code_wat):
@ -44,19 +46,13 @@ def wat2wasm(code_wat):
class SuiteResult:
def __init__(self):
self.log_int32_list = []
self.returned_value = None
def callback_log_int32(self, store, value):
del store # auto passed by pywasm
self.log_int32_list.append(value)
def make_imports(self):
return {
'console': {
'logInt32': self.callback_log_int32,
}
RUNNER_CLASS_MAP = {
'pywasm': runners.RunnerPywasm,
'pywasm3': runners.RunnerPywasm3,
'wasmtime': runners.RunnerWasmtime,
'wasmer': runners.RunnerWasmer,
}
class Suite:
@ -73,189 +69,60 @@ class Suite:
Returned is an object with the results set
"""
phasm_module = phasm_parse(self.code_py)
class_ = RUNNER_CLASS_MAP[runtime]
runner = class_(self.code_py)
runner.parse()
runner.compile_ast()
runner.compile_wat()
runner.compile_wasm()
runner.interpreter_setup()
runner.interpreter_load(imports)
write_header(sys.stderr, 'Phasm')
runner.dump_phasm_code(sys.stderr)
write_header(sys.stderr, 'Assembly')
runner.dump_wasm_wat(sys.stderr)
# Check if code formatting works
assert self.code_py == '\n' + phasm_render(phasm_module) # \n for formatting in tests
assert self.code_py == '\n' + phasm_render(runner.phasm_ast) # \n for formatting in tests
# Compile
wasm_module = phasm_compile(phasm_module)
# runner.call('stdlib.alloc.__init__')
# Render as WebAssembly text
code_wat = wasm_module.to_wat()
wasm_args = []
if args:
write_header(sys.stderr, 'Memory (pre alloc)')
runner.interpreter_dump_memory(sys.stderr)
sys.stderr.write(f'{DASHES} Assembly {DASHES}\n')
_write_numbered_lines(code_wat)
# Compile to assembly code
code_wasm = wat2wasm(code_wat)
# Run assembly code
if 'pywasm' == runtime:
return _run_pywasm(code_wasm, args)
if 'pywasm3' == runtime:
return _run_pywasm3(code_wasm, args)
if 'wasmtime' == runtime:
return _run_wasmtime(code_wasm, args)
if 'wasmer' == runtime:
return _run_wasmer(code_wasm, args, imports)
raise Exception(f'Invalid runtime: {runtime}')
def _run_pywasm(code_wasm, args):
# https://pypi.org/project/pywasm/
result = SuiteResult()
module = pywasm.binary.Module.from_reader(io.BytesIO(code_wasm))
runtime = pywasm.Runtime(module, result.make_imports(), {})
def set_byte(idx, byt):
runtime.store.mems[0].data[idx] = byt
args = _convert_bytes_arguments(
args,
lambda x: runtime.exec('___new_reference___', [x]),
set_byte
)
sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n')
_dump_memory(runtime.store.mems[0].data)
result.returned_value = runtime.exec('testEntry', args)
sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n')
_dump_memory(runtime.store.mems[0].data)
return result
def _run_pywasm3(code_wasm, args):
# https://pypi.org/project/pywasm3/
result = SuiteResult()
env = wasm3.Environment()
mod = env.parse_module(code_wasm)
rtime = env.new_runtime(1024 * 1024)
rtime.load(mod)
def set_byte(idx, byt):
rtime.get_memory(0)[idx] = byt
args = _convert_bytes_arguments(
args,
rtime.find_function('___new_reference___'),
set_byte
)
sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n')
_dump_memory(rtime.get_memory(0))
result.returned_value = rtime.find_function('testEntry')(*args)
sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n')
_dump_memory(rtime.get_memory(0))
return result
def _run_wasmtime(code_wasm, args):
# https://pypi.org/project/wasmtime/
result = SuiteResult()
store = wasmtime.Store()
module = wasmtime.Module(store.engine, code_wasm)
instance = wasmtime.Instance(store, module, [])
sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n')
sys.stderr.write('<Not available on wasmtime>\n')
result.returned_value = instance.exports(store)['testEntry'](store, *args)
sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n')
sys.stderr.write('<Not available on wasmtime>\n')
return result
def _run_wasmer(code_wasm, args, imports):
# https://pypi.org/project/wasmer/
result = SuiteResult()
store = wasmer.Store(wasmer.engine.JIT(wasmer_compiler_cranelift.Compiler))
import_object = wasmer.ImportObject()
import_object.register('imports', {
k: wasmer.Function(store, v)
for k, v in (imports or {}).items()
})
# Let's compile the module to be able to execute it!
module = wasmer.Module(store, code_wasm)
# Now the module is compiled, we can instantiate it.
instance = wasmer.Instance(module, import_object)
sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n')
sys.stderr.write('<Not available on wasmer>\n')
# Call the exported `sum` function.
result.returned_value = instance.exports.testEntry(*args)
sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n')
sys.stderr.write('<Not available on wasmer>\n')
return result
def _convert_bytes_arguments(args, new_reference, set_byte):
result = []
for arg in args:
if not isinstance(arg, bytes):
result.append(arg)
if isinstance(arg, (int, float, )):
wasm_args.append(arg)
continue
if isinstance(arg, bytes):
# TODO: Implement and use the bytes constructor function
offset = new_reference(len(arg) + 4)
result.append(offset)
# TODO: call upon stdlib.alloc.__init__ and stdlib.alloc.__alloc__
adr = runner.call('___new_reference___', len(arg) + 4)
sys.stderr.write(f'Allocation 0x{adr:08x} {repr(arg)}\n')
# Store the length prefix
for idx, byt in enumerate(len(arg).to_bytes(4, byteorder='little')):
set_byte(offset + idx, byt)
runner.interpreter_write_memory(adr, len(arg).to_bytes(4, byteorder='little'))
runner.interpreter_write_memory(adr + 4, arg)
wasm_args.append(adr)
continue
# Store the actual bytes
for idx, byt in enumerate(arg):
set_byte(offset + 4 + idx, byt)
raise NotImplementedError(arg)
write_header(sys.stderr, 'Memory (pre run)')
runner.interpreter_dump_memory(sys.stderr)
result = SuiteResult()
result.returned_value = runner.call('testEntry', *wasm_args)
write_header(sys.stderr, 'Memory (post run)')
runner.interpreter_dump_memory(sys.stderr)
return result
def _dump_memory(mem):
line_width = 16
prev_line = None
skip = False
for idx in range(0, len(mem), line_width):
line = ''
for idx2 in range(0, line_width):
line += f'{mem[idx + idx2]:02X}'
if idx2 % 2 == 1:
line += ' '
if prev_line == line:
if not skip:
sys.stderr.write('**\n')
skip = True
else:
sys.stderr.write(f'{idx:08x} {line}\n')
prev_line = line
def _write_numbered_lines(text: str) -> None:
line_list = text.split('\n')
line_no_width = len(str(len(line_list)))
for line_no, line_txt in enumerate(line_list):
sys.stderr.write('{} {}\n'.format(
str(line_no + 1).zfill(line_no_width),
line_txt,
))
def write_header(textio, msg: str) -> None:
textio.write(f'{DASHES} {msg.ljust(16)} {DASHES}\n')

30
tests/integration/runners.py
View File

@ -1,7 +1,7 @@
"""
Runners to help run WebAssembly code on various interpreters
"""
from typing import Any, Iterable, TextIO
from typing import Any, Callable, Dict, Iterable, Optional, TextIO
import ctypes
import io
@ -71,7 +71,7 @@ class RunnerBase:
"""
raise NotImplementedError
def interpreter_load(self) -> None:
def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None:
"""
Loads the code into the interpreter
"""
@ -114,7 +114,10 @@ class RunnerPywasm(RunnerBase):
# Nothing to set up
pass
def interpreter_load(self) -> None:
def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None:
if imports is not None:
raise NotImplementedError
bytesio = io.BytesIO(self.wasm_bin)
self.module = pywasm.binary.Module.from_reader(bytesio)
self.runtime = pywasm.Runtime(self.module, {}, None)
@ -146,7 +149,10 @@ class RunnerPywasm3(RunnerBase):
self.env = wasm3.Environment()
self.rtime = self.env.new_runtime(1024 * 1024)
def interpreter_load(self) -> None:
def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None:
if imports is not None:
raise NotImplementedError
self.mod = self.env.parse_module(self.wasm_bin)
self.rtime.load(self.mod)
@ -179,7 +185,10 @@ class RunnerWasmtime(RunnerBase):
def interpreter_setup(self) -> None:
self.store = wasmtime.Store()
def interpreter_load(self) -> None:
def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None:
if imports is not None:
raise NotImplementedError
self.module = wasmtime.Module(self.store.engine, self.wasm_bin)
self.instance = wasmtime.Instance(self.store, self.module, [])
@ -242,9 +251,16 @@ class RunnerWasmer(RunnerBase):
def interpreter_setup(self) -> None:
self.store = wasmer.Store()
def interpreter_load(self) -> None:
def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None:
import_object = wasmer.ImportObject()
if imports:
import_object.register('imports', {
k: wasmer.Function(self.store, v)
for k, v in (imports or {}).items()
})
self.module = wasmer.Module(self.store, self.wasm_bin)
self.instance = wasmer.Instance(self.module)
self.instance = wasmer.Instance(self.module, import_object)
def interpreter_write_memory(self, offset: int, data: Iterable[int]) -> None:
exports = self.instance.exports

13
tests/integration/test_simple.py
View File

@ -114,7 +114,6 @@ def testEntry(a: i32) -> i64:
result = Suite(code_py).run_code(125)
assert 125 == result.returned_value
assert [] == result.log_int32_list
@pytest.mark.integration_test
@pytest.mark.skip('Do we want it to work like this?')
@ -128,7 +127,6 @@ def testEntry(a: i32, b: i64) -> i64:
result = Suite(code_py).run_code(125, 100)
assert 225 == result.returned_value
assert [] == result.log_int32_list
@pytest.mark.integration_test
@pytest.mark.skip('Do we want it to work like this?')
@ -142,7 +140,6 @@ def testEntry(a: f32) -> f64:
result = Suite(code_py).run_code(125.5)
assert 125.5 == result.returned_value
assert [] == result.log_int32_list
@pytest.mark.integration_test
@pytest.mark.skip('Do we want it to work like this?')
@ -156,7 +153,6 @@ def testEntry(a: f32, b: f64) -> f64:
result = Suite(code_py).run_code(125.5, 100.25)
assert 225.75 == result.returned_value
assert [] == result.log_int32_list
@pytest.mark.integration_test
@pytest.mark.skip('TODO')
@ -170,7 +166,6 @@ def testEntry() -> i32:
result = Suite(code_py).run_code()
assert 523 == result.returned_value
assert [] == result.log_int32_list
@pytest.mark.integration_test
@pytest.mark.skip('TODO')
@ -184,7 +179,6 @@ def testEntry() -> i32:
result = Suite(code_py).run_code()
assert -19 == result.returned_value
assert [] == result.log_int32_list
@pytest.mark.integration_test
@pytest.mark.parametrize('inp', [9, 10, 11, 12])
@ -268,7 +262,6 @@ def testEntry() -> i32:
result = Suite(code_py).run_code()
assert 13 == result.returned_value
assert [] == result.log_int32_list
@pytest.mark.integration_test
def test_call_post_defined():
@ -284,7 +277,6 @@ def helper(left: i32, right: i32) -> i32:
result = Suite(code_py).run_code()
assert 7 == result.returned_value
assert [] == result.log_int32_list
@pytest.mark.integration_test
@pytest.mark.parametrize('type_', COMPLETE_SIMPLE_TYPES)
@ -317,7 +309,6 @@ def testEntry() -> i32:
result = Suite(code_py).run_code()
assert 8947 == result.returned_value
assert [] == result.log_int32_list
@pytest.mark.integration_test
@pytest.mark.parametrize('type_', TYPE_MAP.keys())
@ -337,7 +328,6 @@ def helper(cv: CheckedValue) -> {type_}:
result = Suite(code_py).run_code()
assert 23 == result.returned_value
assert [] == result.log_int32_list
@pytest.mark.integration_test
def test_struct_1():
@ -358,7 +348,6 @@ def helper(shape: Rectangle) -> i32:
result = Suite(code_py).run_code()
assert 252 == result.returned_value
assert [] == result.log_int32_list
@pytest.mark.integration_test
def test_struct_2():
@ -379,7 +368,6 @@ def helper(shape1: Rectangle, shape2: Rectangle) -> i32:
result = Suite(code_py).run_code()
assert 545 == result.returned_value
assert [] == result.log_int32_list
@pytest.mark.integration_test
@pytest.mark.parametrize('type_', COMPLETE_SIMPLE_TYPES)
@ -412,7 +400,6 @@ def helper(v: (f32, f32, f32, )) -> f32:
result = Suite(code_py).run_code()
assert 3.74 < result.returned_value < 3.75
assert [] == result.log_int32_list
@pytest.mark.integration_test
def test_bytes_address():

16
tests/integration/test_stdlib_alloc.py
View File

@ -2,7 +2,7 @@ import sys
import pytest
from .helpers import DASHES
from .helpers import write_header
from .runners import RunnerPywasm3 as Runner
def setup_interpreter(phash_code: str) -> Runner:
@ -15,9 +15,9 @@ def setup_interpreter(phash_code: str) -> Runner:
runner.interpreter_setup()
runner.interpreter_load()
sys.stderr.write(f'{DASHES} Phasm {DASHES}\n')
write_header(sys.stderr, 'Phasm')
runner.dump_phasm_code(sys.stderr)
sys.stderr.write(f'{DASHES} Assembly {DASHES}\n')
write_header(sys.stderr, 'Assembly')
runner.dump_wasm_wat(sys.stderr)
return runner
@ -35,12 +35,12 @@ def testEntry() -> u8:
# Garbage in the memory so we can test for it
runner.interpreter_write_memory(0, range(128))
sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n')
write_header(sys.stderr, 'Memory (pre run)')
runner.interpreter_dump_memory(sys.stderr)
runner.call('stdlib.alloc.__init__')
sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n')
write_header(sys.stderr, 'Memory (post run)')
runner.interpreter_dump_memory(sys.stderr)
assert (
@ -61,7 +61,7 @@ def testEntry() -> u8:
runner = setup_interpreter(code_py)
sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n')
write_header(sys.stderr, 'Memory (pre run)')
runner.interpreter_dump_memory(sys.stderr)
with pytest.raises(Exception, match='unreachable'):
@ -77,7 +77,7 @@ def testEntry() -> u8:
runner = setup_interpreter(code_py)
sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n')
write_header(sys.stderr, 'Memory (pre run)')
runner.interpreter_dump_memory(sys.stderr)
runner.call('stdlib.alloc.__init__')
@ -85,7 +85,7 @@ def testEntry() -> u8:
offset1 = runner.call('stdlib.alloc.__alloc__', 32)
offset2 = runner.call('stdlib.alloc.__alloc__', 32)
sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n')
write_header(sys.stderr, 'Memory (post run)')
runner.interpreter_dump_memory(sys.stderr)
assert (