From 473cd0b6263c35370110468618ba932b9bf23498 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 3 Apr 2021 12:45:13 +0200 Subject: [PATCH 01/73] cleanups [skip-ci] --- .gitignore | 2 ++ Makefile | 2 +- README.md | 4 ++++ compile.py | 27 +++++++++++++++++++++++++-- log.py | 5 +++-- 5 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 .gitignore create mode 100644 README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..50765be --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.wasm +*.wat diff --git a/Makefile b/Makefile index 58bd1f5..7bbb8fb 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ %.wat: %.py compile.py - python3.8 compile.py $< > $@ + python3.8 compile.py $< $@ %.wasm: %.wat wat2wasm $^ -o $@ diff --git a/README.md b/README.md new file mode 100644 index 0000000..4fb6851 --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +webasm (python) +=============== + +You will need wat2wasm from github.com/WebAssembly/wabt in your path. diff --git a/compile.py b/compile.py index 6720a4a..2fcaf84 100644 --- a/compile.py +++ b/compile.py @@ -52,6 +52,28 @@ class Visitor(ast.NodeVisitor): )) def visit_FunctionDef(self, node): + if node.decorator_list: + # TODO: Support normal decorators + assert 1 == len(node.decorator_list) + call = node.decorator_list[0] + assert 'external' == call.func.id + assert 1 == len(call.args) + assert isinstance(call.args[0].value, str) + + import_ = Import( + call.args[0].value, + node.name, + node.name, + ) + + import_.params = [ + arg.annotation.id + for arg in node.args.args + ] + + self.imports.append(import_) + return + func = Function( node.name, ) @@ -116,7 +138,7 @@ class Visitor(ast.NodeVisitor): def err(msg: str) -> None: sys.stderr.write('{}\n'.format(msg)) -def main(source: str) -> int: +def main(source: str, sink: str) -> int: with open(source, 'r') as fil: code = fil.read() @@ -125,7 +147,8 @@ def main(source: str) -> int: visitor = Visitor() visitor.visit(res) - print(visitor.generate()) + with open(sink, 'w') as fil: + fil.write(visitor.generate()) return 0 diff --git a/log.py b/log.py index 85f29fd..6985c05 100644 --- a/log.py +++ b/log.py @@ -1,5 +1,6 @@ -from console import log as log -"log(i32)" +@external('console') +def log(msg: i32) -> None: + ... def logIt(): log(13 + 13 * 123) From 2a93d6125ebd7c6b0ec4203f5da1608233e43457 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 3 Apr 2021 18:37:09 +0200 Subject: [PATCH 02/73] ideas [skip-ci] --- .gitignore | 7 ++- Makefile | 8 ++++ compile.py | 17 ++++--- func.html | 20 --------- log.html | 29 ------------ log.py | 6 --- requirements.txt | 2 + tests/__init__.py | 0 tests/integration/__init__.py | 0 tests/integration/test_fib.py | 85 +++++++++++++++++++++++++++++++++++ 10 files changed, 111 insertions(+), 63 deletions(-) delete mode 100644 func.html delete mode 100644 log.html delete mode 100644 log.py create mode 100644 requirements.txt create mode 100644 tests/__init__.py create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/test_fib.py diff --git a/.gitignore b/.gitignore index 50765be..7f7c00b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ -*.wasm -*.wat +/*.wasm +/*.wat +/venv + +__pycache__ diff --git a/Makefile b/Makefile index 7bbb8fb..3ed270c 100644 --- a/Makefile +++ b/Makefile @@ -6,3 +6,11 @@ server: python3.8 -m http.server + +test: venv/.done + venv/bin/pytest tests + +venv/.done: requirements.txt + python3.8 -m venv venv + venv/bin/python3 -m pip install -r $^ + touch $@ diff --git a/compile.py b/compile.py index 2fcaf84..f2c6119 100644 --- a/compile.py +++ b/compile.py @@ -138,17 +138,22 @@ class Visitor(ast.NodeVisitor): def err(msg: str) -> None: sys.stderr.write('{}\n'.format(msg)) -def main(source: str, sink: str) -> int: - with open(source, 'r') as fil: - code = fil.read() - - res = ast.parse(code, source) +def process(input: str, input_name: str) -> str: + res = ast.parse(input, input_name) visitor = Visitor() visitor.visit(res) + return visitor.generate() + +def main(source: str, sink: str) -> int: + with open(source, 'r') as fil: + code_py = fil.read() + + code_wat = process(code_py, source) + with open(sink, 'w') as fil: - fil.write(visitor.generate()) + fil.write(code_wat) return 0 diff --git a/func.html b/func.html deleted file mode 100644 index 4967ac5..0000000 --- a/func.html +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - Simple add example - - - - - - - diff --git a/log.html b/log.html deleted file mode 100644 index 8f97d5d..0000000 --- a/log.html +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - Simple log example - - - - - - - diff --git a/log.py b/log.py deleted file mode 100644 index 6985c05..0000000 --- a/log.py +++ /dev/null @@ -1,6 +0,0 @@ -@external('console') -def log(msg: i32) -> None: - ... - -def logIt(): - log(13 + 13 * 123) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9e01a86 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +pywasm==1.0.7 +pytest==6.2.2 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/test_fib.py b/tests/integration/test_fib.py new file mode 100644 index 0000000..bcd4719 --- /dev/null +++ b/tests/integration/test_fib.py @@ -0,0 +1,85 @@ +import io +import subprocess +import sys +from tempfile import NamedTemporaryFile + +from pywasm import binary +from pywasm import Runtime + +from compile import process + +def wat2wasm(code_wat): + with NamedTemporaryFile('w+t') as input_fp: + input_fp.write(code_wat) + input_fp.flush() + + with NamedTemporaryFile('w+b') as output_fp: + result = subprocess.run( + [ + 'wat2wasm', + input_fp.name, + '-o', + output_fp.name, + ], + check=True, + ) + + output_fp.seek(0) + + return output_fp.read() + +class SuiteResult: + def __init__(self): + self.log_int32_list = [] + + def callback_log_int32(self, store, value): + del store # auto passed by pywasm + + self.log_int32_list.append(value) + + def make_imports(self): + return { + 'console': { + 'logInt32': self.callback_log_int32, + } + } + +class Suite: + def __init__(self, code_py, test_name): + self.code_py = code_py + self.test_name = test_name + + def run_code(self, *args): + """ + Compiles the given python code into wasm and + then runs it + + Returned is an object with the results set + """ + code_wat = process(self.code_py, self.test_name) + sys.stderr.write(code_wat) + + code_wasm = wat2wasm(code_wat) + module = binary.Module.from_reader(io.BytesIO(code_wasm)) + + result = SuiteResult() + runtime = Runtime(module, result.make_imports(), {}) + + result.returned_value = runtime.exec('testEntry', args) + + return result + +def test_fib(): + code_py = """ +@external('console') +def logInt32(value: i32) -> None: + ... + +def testEntry(): + logInt32(13 + 13 * 123) +""" + + result = Suite(code_py, 'test_fib').run_code() + + assert None is result.returned_value + assert [1612] == result.log_int32_list From edd12e5b7ce640a962acc457cec228ad24289ef6 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sun, 4 Apr 2021 16:32:56 +0200 Subject: [PATCH 03/73] cleanup [skip-ci] --- compile.py | 143 ++------------------------------------------ py2wasm/__init__.py | 0 py2wasm/python.py | 94 +++++++++++++++++++++++++++++ py2wasm/wasm.py | 45 ++++++++++++++ 4 files changed, 145 insertions(+), 137 deletions(-) create mode 100644 py2wasm/__init__.py create mode 100644 py2wasm/python.py create mode 100644 py2wasm/wasm.py diff --git a/compile.py b/compile.py index f2c6119..50a6168 100644 --- a/compile.py +++ b/compile.py @@ -2,149 +2,18 @@ import _ast import ast import sys -class Import: - def __init__(self, module, name, intname): - self.module = module - self.name = name - self.intname = intname - self.params = None - - def generate(self): - return '(import "{}" "{}" (func ${}{}))'.format( - self.module, - self.name, - self.intname, - ''.join(' (param {})'.format(x) for x in self.params) - ) - -class Statement: - def __init__(self, name, *args): - self.name = name - self.args = args - - def generate(self): - return '{} {}'.format(self.name, ' '.join(self.args)) - -class Function: - def __init__(self, name, exported=True): - self.name = name - self.exported = exported # TODO: Use __all__! - self.statements = [] - - def generate(self): - return '(func {}\n {})'.format( - ('(export "{}")' if self.exported else '${}').format(self.name), - '\n '.join(x.generate() for x in self.statements), - ) - -class Visitor(ast.NodeVisitor): - def __init__(self): - self._stack = [] - self.imports = [] - self.functions = [] - - def visit_ImportFrom(self, node): - for alias in node.names: - self.imports.append(Import( - node.module, - alias.name, - alias.asname, - )) - - def visit_FunctionDef(self, node): - if node.decorator_list: - # TODO: Support normal decorators - assert 1 == len(node.decorator_list) - call = node.decorator_list[0] - assert 'external' == call.func.id - assert 1 == len(call.args) - assert isinstance(call.args[0].value, str) - - import_ = Import( - call.args[0].value, - node.name, - node.name, - ) - - import_.params = [ - arg.annotation.id - for arg in node.args.args - ] - - self.imports.append(import_) - return - - func = Function( - node.name, - ) - - self._stack.append(func) - self.generic_visit(node) - self._stack.pop() - - self.functions.append(func) - - def visit_Expr(self, node): - self.generic_visit(node) - - def visit_Call(self, node): - self.generic_visit(node) - - func = self._stack[-1] - func.statements.append( - Statement('call', '$' + node.func.id) - ) - - def visit_BinOp(self, node): - self.generic_visit(node) - - func = self._stack[-1] - - if 'Add' == node.op.__class__.__name__: - func.statements.append( - Statement('i32.add') - ) - elif 'Mult' == node.op.__class__.__name__: - func.statements.append( - Statement('i32.mul') - ) - else: - err(node.op) - - def visit_Constant(self, node): - if not self._stack: - # Constant outside of any function - imp = self.imports[-1] - prefix = imp.name + '(' - val = node.value.strip() - - if val.startswith(prefix) and val.endswith(')'): - imp.params = val[len(prefix):-1].split(',') - else: - func = self._stack[-1] - if isinstance(node.value, int): - func.statements.append( - Statement('i32.const', str(node.value)) - ) - - self.generic_visit(node) - - def generate(self): - return '(module\n {}\n {})'.format( - '\n '.join(x.generate() for x in self.imports), - '\n '.join(x.generate() for x in self.functions), - ) - -def err(msg: str) -> None: - sys.stderr.write('{}\n'.format(msg)) +from py2wasm.wasm import Module +from py2wasm.python import Visitor def process(input: str, input_name: str) -> str: res = ast.parse(input, input_name) - visitor = Visitor() + module = Module() + + visitor = Visitor(module) visitor.visit(res) - return visitor.generate() + return module.generate() def main(source: str, sink: str) -> int: with open(source, 'r') as fil: diff --git a/py2wasm/__init__.py b/py2wasm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/py2wasm/python.py b/py2wasm/python.py new file mode 100644 index 0000000..6be9b1e --- /dev/null +++ b/py2wasm/python.py @@ -0,0 +1,94 @@ +import ast + +from .wasm import Function, Import, Module, Statement + +class Visitor(ast.NodeVisitor): + def __init__(self, module: Module) -> None: + self._stack = [] + self.module = module + + # def visit_ImportFrom(self, node): + # for alias in node.names: + # self.imports.append(Import( + # node.module, + # alias.name, + # alias.asname, + # )) + + def visit_FunctionDef(self, node): + if node.decorator_list: + # TODO: Support normal decorators + assert 1 == len(node.decorator_list) + call = node.decorator_list[0] + assert 'external' == call.func.id + assert 1 == len(call.args) + assert isinstance(call.args[0].value, str) + + import_ = Import( + call.args[0].value, + node.name, + node.name, + ) + + import_.params = [ + arg.annotation.id + for arg in node.args.args + ] + + self.module.imports.append(import_) + return + + func = Function( + node.name, + ) + + self._stack.append(func) + self.generic_visit(node) + self._stack.pop() + + self.module.functions.append(func) + + def visit_Expr(self, node): + self.generic_visit(node) + + def visit_Call(self, node): + self.generic_visit(node) + + func = self._stack[-1] + func.statements.append( + Statement('call', '$' + node.func.id) + ) + + def visit_BinOp(self, node): + self.generic_visit(node) + + func = self._stack[-1] + + if 'Add' == node.op.__class__.__name__: + func.statements.append( + Statement('i32.add') + ) + elif 'Mult' == node.op.__class__.__name__: + func.statements.append( + Statement('i32.mul') + ) + else: + raise NotImplementedError + + def visit_Constant(self, node): + if not self._stack: + # Constant outside of any function + imp = self.imports[-1] + prefix = imp.name + '(' + val = node.value.strip() + + if val.startswith(prefix) and val.endswith(')'): + imp.params = val[len(prefix):-1].split(',') + else: + func = self._stack[-1] + if isinstance(node.value, int): + func.statements.append( + Statement('i32.const', str(node.value)) + ) + + self.generic_visit(node) diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py new file mode 100644 index 0000000..f0a3b32 --- /dev/null +++ b/py2wasm/wasm.py @@ -0,0 +1,45 @@ +class Import: + def __init__(self, module, name, intname): + self.module = module + self.name = name + self.intname = intname + self.params = None + + def generate(self): + return '(import "{}" "{}" (func ${}{}))'.format( + self.module, + self.name, + self.intname, + ''.join(' (param {})'.format(x) for x in self.params) + ) + +class Statement: + def __init__(self, name, *args): + self.name = name + self.args = args + + def generate(self): + return '{} {}'.format(self.name, ' '.join(self.args)) + +class Function: + def __init__(self, name, exported=True): + self.name = name + self.exported = exported # TODO: Use __all__! + self.statements = [] + + def generate(self): + return '(func {}\n {})'.format( + ('(export "{}")' if self.exported else '${}').format(self.name), + '\n '.join(x.generate() for x in self.statements), + ) + +class Module: + def __init__(self, imports = None, functions = None) -> None: + self.imports = imports or [] + self.functions = functions or [] + + def generate(self): + return '(module\n {}\n {})'.format( + '\n '.join(x.generate() for x in self.imports), + '\n '.join(x.generate() for x in self.functions), + ) From 633267b37df426b728f220a1cf1e2504c9b2a382 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sun, 4 Apr 2021 16:43:55 +0200 Subject: [PATCH 04/73] ideas [skip-ci] --- py2wasm/python.py | 43 +++++++++++++++++++++++------------ py2wasm/wasm.py | 3 ++- tests/integration/test_fib.py | 6 ++++- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/py2wasm/python.py b/py2wasm/python.py index 6be9b1e..7566cbf 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -16,32 +16,47 @@ class Visitor(ast.NodeVisitor): # )) def visit_FunctionDef(self, node): + is_export = False + if node.decorator_list: # TODO: Support normal decorators assert 1 == len(node.decorator_list) + call = node.decorator_list[0] - assert 'external' == call.func.id - assert 1 == len(call.args) - assert isinstance(call.args[0].value, str) + if not isinstance(call, ast.Name): + assert isinstance(call, ast.Call) - import_ = Import( - call.args[0].value, - node.name, - node.name, - ) + assert 'external' == call.func.id + assert 1 == len(call.args) + assert isinstance(call.args[0].value, str) - import_.params = [ - arg.annotation.id - for arg in node.args.args - ] + import_ = Import( + call.args[0].value, + node.name, + node.name, + ) - self.module.imports.append(import_) - return + import_.params = [ + arg.annotation.id + for arg in node.args.args + ] + + self.module.imports.append(import_) + return + + assert call.id == 'export' + is_export = True func = Function( node.name, + is_export, ) + for arg in node.args.args: + func.params.append( + + ) + self._stack.append(func) self.generic_visit(node) self._stack.pop() diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py index f0a3b32..ce6f81f 100644 --- a/py2wasm/wasm.py +++ b/py2wasm/wasm.py @@ -25,6 +25,7 @@ class Function: def __init__(self, name, exported=True): self.name = name self.exported = exported # TODO: Use __all__! + self.params = [] self.statements = [] def generate(self): @@ -39,7 +40,7 @@ class Module: self.functions = functions or [] def generate(self): - return '(module\n {}\n {})'.format( + return '(module\n {}\n {})\n'.format( '\n '.join(x.generate() for x in self.imports), '\n '.join(x.generate() for x in self.functions), ) diff --git a/tests/integration/test_fib.py b/tests/integration/test_fib.py index bcd4719..16fa464 100644 --- a/tests/integration/test_fib.py +++ b/tests/integration/test_fib.py @@ -75,8 +75,12 @@ def test_fib(): def logInt32(value: i32) -> None: ... +def helper(arg: i32) -> i32: + return arg + 1 + +@export def testEntry(): - logInt32(13 + 13 * 123) + logInt32(13 + 13 * helper(122)) """ result = Suite(code_py, 'test_fib').run_code() From 58bd5f088959c81e7a25aaf63de8fa14d1e0214a Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Mon, 5 Apr 2021 10:41:26 +0200 Subject: [PATCH 05/73] More cleanup [skip-ci] --- Makefile | 10 +- compile.py | 30 ---- py2wasm/__main__.py | 25 +++ py2wasm/python.py | 311 +++++++++++++++++++++++----------- py2wasm/utils.py | 18 ++ py2wasm/wasm.py | 74 ++++++-- pylintrc | 2 + requirements.txt | 4 +- tests/integration/helpers.py | 77 +++++++++ tests/integration/test_fib.py | 79 +-------- 10 files changed, 409 insertions(+), 221 deletions(-) delete mode 100644 compile.py create mode 100644 py2wasm/__main__.py create mode 100644 py2wasm/utils.py create mode 100644 pylintrc create mode 100644 tests/integration/helpers.py diff --git a/Makefile b/Makefile index 3ed270c..3633841 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -%.wat: %.py compile.py - python3.8 compile.py $< $@ +%.wat: %.py py2wasm/*.py + python3.8 -m py2wasm $< $@ %.wasm: %.wat wat2wasm $^ -o $@ @@ -10,6 +10,12 @@ server: test: venv/.done venv/bin/pytest tests +lint: venv/.done + venv/bin/pylint py2wasm + +typecheck: venv/.done + venv/bin/mypy --strict py2wasm + venv/.done: requirements.txt python3.8 -m venv venv venv/bin/python3 -m pip install -r $^ diff --git a/compile.py b/compile.py deleted file mode 100644 index 50a6168..0000000 --- a/compile.py +++ /dev/null @@ -1,30 +0,0 @@ -import _ast -import ast -import sys - -from py2wasm.wasm import Module -from py2wasm.python import Visitor - -def process(input: str, input_name: str) -> str: - res = ast.parse(input, input_name) - - module = Module() - - visitor = Visitor(module) - visitor.visit(res) - - return module.generate() - -def main(source: str, sink: str) -> int: - with open(source, 'r') as fil: - code_py = fil.read() - - code_wat = process(code_py, source) - - with open(sink, 'w') as fil: - fil.write(code_wat) - - return 0 - -if __name__ == '__main__': - sys.exit(main(*sys.argv[1:])) diff --git a/py2wasm/__main__.py b/py2wasm/__main__.py new file mode 100644 index 0000000..59757dd --- /dev/null +++ b/py2wasm/__main__.py @@ -0,0 +1,25 @@ +""" +Functions for using this module from CLI +""" + +import sys + +from .utils import process + +def main(source: str, sink: str) -> int: + """ + Main method + """ + + with open(source, 'r') as fil: + code_py = fil.read() + + code_wat = process(code_py, source) + + with open(sink, 'w') as fil: + fil.write(code_wat) + + return 0 + +if __name__ == '__main__': + sys.exit(main(*sys.argv[1:])) # pylint: disable=E1120 diff --git a/py2wasm/python.py b/py2wasm/python.py index 7566cbf..da7c140 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -1,11 +1,134 @@ +""" +Code for parsing the source (python-alike) code +""" + +from typing import List, Optional, Union, Tuple + import ast -from .wasm import Function, Import, Module, Statement +from . import wasm + +# pylint: disable=C0103,R0201 + +class Visitor: + """ + Class to visit a Python syntax tree + + Since we need to visit the whole tree, there's no point in subclassing + the buildin visitor + """ + def visit_Module(self, node: ast.Module) -> wasm.Module: + """ + Visits a Python module, which results in a wasm Module + """ + assert not node.type_ignores + + module = wasm.Module() + + for stmt in node.body: + if isinstance(stmt, ast.FunctionDef): + wnode = self.visit_FunctionDef(stmt) + + if isinstance(wnode, wasm.Import): + module.imports.append(wnode) + else: + module.functions.append(wnode) + continue + + raise NotImplementedError(stmt) + + return module + + def visit_FunctionDef(self, node: ast.FunctionDef) -> Union[wasm.Import, wasm.Function]: + """ + A Python function definition with the @external decorator is returned as import. + + Other functions are returned as (exported) functions. + """ + + exported = False + + if node.decorator_list: + assert 1 == len(node.decorator_list) + + decorator = node.decorator_list[0] + if isinstance(decorator, ast.Name): + assert 'exported' == decorator.id + exported = True + elif isinstance(decorator, ast.Call): + call = decorator + + assert not node.type_comment + assert 1 == len(node.body) + assert isinstance(node.body[0], ast.Expr) + assert isinstance(node.body[0].value, ast.Ellipsis) + + assert isinstance(call.func, ast.Name) + assert 'imported' == call.func.id + + assert not call.keywords + + module, name, intname = _parse_import_decorator(node.name, call.args) + + return wasm.Import(module, name, intname, _parse_import_args(node.args)) + + return wasm.Function(node.name, exported) + +def _parse_import_decorator(func_name: str, args: List[ast.expr]) -> Tuple[str, str, str]: + """ + Parses an @import decorator + """ + + assert 0 < len(args) < 3 + str_args = [ + arg.value + for arg in args + if isinstance(arg, ast.Constant) + and isinstance(arg.value, str) + ] + assert len(str_args) == len(args) + + module = str_args.pop(0) + if str_args: + name = str_args.pop(0) + else: + name = func_name + + return module, name, func_name + +def _parse_import_args(args: ast.arguments) -> List[Tuple[str, str]]: + """ + Parses the arguments for an @imported method + """ + + assert not args.vararg + assert not args.kwonlyargs + assert not args.kw_defaults + assert not args.kwarg + assert not args.defaults # Maybe support this later on + + arg_list = [ + *args.posonlyargs, + *args.args, + ] + + return [ + (arg.arg, _parse_annotation(arg.annotation)) + for arg in arg_list + ] + +def _parse_annotation(ann: Optional[ast.expr]) -> str: + """ + Parses a typing annotation + """ + assert ann is not None, 'Web Assembly requires type annotations' + assert isinstance(ann, ast.Name) + result = ann.id + assert result in ['i32'] + return result + + -class Visitor(ast.NodeVisitor): - def __init__(self, module: Module) -> None: - self._stack = [] - self.module = module # def visit_ImportFrom(self, node): # for alias in node.names: @@ -14,96 +137,88 @@ class Visitor(ast.NodeVisitor): # alias.name, # alias.asname, # )) - - def visit_FunctionDef(self, node): - is_export = False - - if node.decorator_list: - # TODO: Support normal decorators - assert 1 == len(node.decorator_list) - - call = node.decorator_list[0] - if not isinstance(call, ast.Name): - assert isinstance(call, ast.Call) - - assert 'external' == call.func.id - assert 1 == len(call.args) - assert isinstance(call.args[0].value, str) - - import_ = Import( - call.args[0].value, - node.name, - node.name, - ) - - import_.params = [ - arg.annotation.id - for arg in node.args.args - ] - - self.module.imports.append(import_) - return - - assert call.id == 'export' - is_export = True - - func = Function( - node.name, - is_export, - ) - - for arg in node.args.args: - func.params.append( - - ) - - self._stack.append(func) - self.generic_visit(node) - self._stack.pop() - - self.module.functions.append(func) - - def visit_Expr(self, node): - self.generic_visit(node) - - def visit_Call(self, node): - self.generic_visit(node) - - func = self._stack[-1] - func.statements.append( - Statement('call', '$' + node.func.id) - ) - - def visit_BinOp(self, node): - self.generic_visit(node) - - func = self._stack[-1] - - if 'Add' == node.op.__class__.__name__: - func.statements.append( - Statement('i32.add') - ) - elif 'Mult' == node.op.__class__.__name__: - func.statements.append( - Statement('i32.mul') - ) - else: - raise NotImplementedError - - def visit_Constant(self, node): - if not self._stack: - # Constant outside of any function - imp = self.imports[-1] - prefix = imp.name + '(' - val = node.value.strip() - - if val.startswith(prefix) and val.endswith(')'): - imp.params = val[len(prefix):-1].split(',') - else: - func = self._stack[-1] - if isinstance(node.value, int): - func.statements.append( - Statement('i32.const', str(node.value)) - ) - - self.generic_visit(node) + # + # def generic_visit(self, node: Any) -> None: + # print(node) + # super().generic_visit(node) + # + # def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + # is_export = False + # + # if node.decorator_list: + # assert 1 == len(node.decorator_list) + # + # call = node.decorator_list[0] + # if not isinstance(call, ast.Name): + # assert isinstance(call, ast.Call) + # + # assert 'external' == call.func.id + # assert 1 == len(call.args) + # assert isinstance(call.args[0].value, str) + # + # import_ = Import( + # call.args[0].value, + # node.name, + # node.name, + # ) + # + # import_.params = [ + # arg.annotation.id + # for arg in node.args.args + # ] + # + # self.module.imports.append(import_) + # return + # + # assert call.id == 'export' + # is_export = True + # + # func = Function( + # node.name, + # is_export, + # ) + # + # for arg in node.args.args: + # func.params.append( + # (arg.arg, arg.annotation.id) + # ) + # func.result = node.returns + # + # self._stack.append(func) + # self.generic_visit(node) + # self._stack.pop() + # + # self.module.functions.append(func) + # + # def visit_Call(self, node: ast.Call) -> None: + # self.generic_visit(node) + # + # func = self._stack[-1] + # func.statements.append( + # Statement('call', '$' + node.func.id) + # ) + # + # def visit_BinOp(self, node: ast.BinOp) -> None: + # self.generic_visit(node) + # + # func = self._stack[-1] + # + # if 'Add' == node.op.__class__.__name__: + # func.statements.append( + # Statement('i32.add') + # ) + # elif 'Mult' == node.op.__class__.__name__: + # func.statements.append( + # Statement('i32.mul') + # ) + # else: + # raise NotImplementedError + # + # def visit_Constant(self, node: ast.Constant) -> None: + # func = self._stack[-1] + # if isinstance(node.value, int): + # func.statements.append( + # Statement('i32.const', str(node.value)) + # ) + # + # self.generic_visit(node) diff --git a/py2wasm/utils.py b/py2wasm/utils.py new file mode 100644 index 0000000..c1e9b72 --- /dev/null +++ b/py2wasm/utils.py @@ -0,0 +1,18 @@ +""" +Utility functions +""" + +import ast + +from py2wasm.python import Visitor + +def process(source: str, input_name: str) -> str: + """ + Processes the python code into web assembly code + """ + res = ast.parse(source, input_name) + + visitor = Visitor() + module = visitor.visit_Module(res) + + return module.generate() diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py index ce6f81f..0843a28 100644 --- a/py2wasm/wasm.py +++ b/py2wasm/wasm.py @@ -1,45 +1,87 @@ +""" +Python classes for storing the representation of Web Assembly code +""" + +from typing import Iterable, List, Tuple + class Import: - def __init__(self, module, name, intname): + """ + Represents a Web Assembly import + """ + def __init__( + self, + module: str, + name: str, + intname: str, + params: Iterable[Tuple[str, str]], + ) -> None: self.module = module self.name = name self.intname = intname - self.params = None + self.params = [*params] - def generate(self): + def generate(self) -> str: + """ + Generates the text version + """ return '(import "{}" "{}" (func ${}{}))'.format( self.module, self.name, self.intname, - ''.join(' (param {})'.format(x) for x in self.params) + ''.join(' (param {})'.format(x[1]) for x in self.params) ) class Statement: - def __init__(self, name, *args): + """ + Represents a Web Assembly statement + """ + def __init__(self, name: str, *args: str): self.name = name self.args = args - def generate(self): + def generate(self) -> str: + """ + Generates the text version + """ return '{} {}'.format(self.name, ' '.join(self.args)) class Function: - def __init__(self, name, exported=True): + """ + Represents a Web Assembly function + """ + def __init__(self, name: str, exported: bool = True) -> None: self.name = name - self.exported = exported # TODO: Use __all__! - self.params = [] - self.statements = [] + self.exported = exported + self.params: List[Tuple[str, str]] = [] + self.returns = None + self.statements: List[Statement] = [] + + def generate(self) -> str: + """ + Generates the text version + """ + header = ('(export "{}")' if self.exported else '${}').format(self.name) + + for nam, typ in self.params: + header += ' (param ${} {})'.format(nam, typ) - def generate(self): return '(func {}\n {})'.format( - ('(export "{}")' if self.exported else '${}').format(self.name), + header, '\n '.join(x.generate() for x in self.statements), ) class Module: - def __init__(self, imports = None, functions = None) -> None: - self.imports = imports or [] - self.functions = functions or [] + """ + Represents a Web Assembly module + """ + def __init__(self) -> None: + self.imports: List[Import] = [] + self.functions: List[Function] = [] - def generate(self): + def generate(self) -> str: + """ + Generates the text version + """ return '(module\n {}\n {})\n'.format( '\n '.join(x.generate() for x in self.imports), '\n '.join(x.generate() for x in self.functions), diff --git a/pylintrc b/pylintrc new file mode 100644 index 0000000..72d74db --- /dev/null +++ b/pylintrc @@ -0,0 +1,2 @@ +[MASTER] +disable=C0122,R0903 diff --git a/requirements.txt b/requirements.txt index 9e01a86..ef8699b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ -pywasm==1.0.7 +mypy==0.812 +pylint==2.7.4 pytest==6.2.2 +pywasm==1.0.7 diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py new file mode 100644 index 0000000..57ed359 --- /dev/null +++ b/tests/integration/helpers.py @@ -0,0 +1,77 @@ +import io +import subprocess +import sys +from tempfile import NamedTemporaryFile + +from pywasm import binary +from pywasm import Runtime + +from py2wasm.utils import process + +def wat2wasm(code_wat): + with NamedTemporaryFile('w+t') as input_fp: + input_fp.write(code_wat) + input_fp.flush() + + with NamedTemporaryFile('w+b') as output_fp: + result = subprocess.run( + [ + 'wat2wasm', + input_fp.name, + '-o', + output_fp.name, + ], + check=True, + ) + + output_fp.seek(0) + + return output_fp.read() + +class SuiteResult: + def __init__(self): + self.log_int32_list = [] + + def callback_log_int32(self, store, value): + del store # auto passed by pywasm + + self.log_int32_list.append(value) + + def make_imports(self): + return { + 'console': { + 'logInt32': self.callback_log_int32, + } + } + +class Suite: + def __init__(self, code_py, test_name): + self.code_py = code_py + self.test_name = test_name + + def run_code(self, *args): + """ + Compiles the given python code into wasm and + then runs it + + Returned is an object with the results set + """ + code_wat = process(self.code_py, self.test_name) + + line_list = code_wat.split('\n') + line_no_width = len(str(len(line_list))) + for line_no, line_txt in enumerate(line_list): + sys.stderr.write('{} {}\n'.format( + str(line_no + 1).zfill(line_no_width), + line_txt, + )) + + code_wasm = wat2wasm(code_wat) + module = binary.Module.from_reader(io.BytesIO(code_wasm)) + + result = SuiteResult() + runtime = Runtime(module, result.make_imports(), {}) + + result.returned_value = runtime.exec('testEntry', args) + + return result diff --git a/tests/integration/test_fib.py b/tests/integration/test_fib.py index 16fa464..fa1eaf9 100644 --- a/tests/integration/test_fib.py +++ b/tests/integration/test_fib.py @@ -1,84 +1,15 @@ -import io -import subprocess -import sys -from tempfile import NamedTemporaryFile - -from pywasm import binary -from pywasm import Runtime - -from compile import process - -def wat2wasm(code_wat): - with NamedTemporaryFile('w+t') as input_fp: - input_fp.write(code_wat) - input_fp.flush() - - with NamedTemporaryFile('w+b') as output_fp: - result = subprocess.run( - [ - 'wat2wasm', - input_fp.name, - '-o', - output_fp.name, - ], - check=True, - ) - - output_fp.seek(0) - - return output_fp.read() - -class SuiteResult: - def __init__(self): - self.log_int32_list = [] - - def callback_log_int32(self, store, value): - del store # auto passed by pywasm - - self.log_int32_list.append(value) - - def make_imports(self): - return { - 'console': { - 'logInt32': self.callback_log_int32, - } - } - -class Suite: - def __init__(self, code_py, test_name): - self.code_py = code_py - self.test_name = test_name - - def run_code(self, *args): - """ - Compiles the given python code into wasm and - then runs it - - Returned is an object with the results set - """ - code_wat = process(self.code_py, self.test_name) - sys.stderr.write(code_wat) - - code_wasm = wat2wasm(code_wat) - module = binary.Module.from_reader(io.BytesIO(code_wasm)) - - result = SuiteResult() - runtime = Runtime(module, result.make_imports(), {}) - - result.returned_value = runtime.exec('testEntry', args) - - return result +from .helpers import Suite def test_fib(): code_py = """ -@external('console') +@imported('console') def logInt32(value: i32) -> None: ... -def helper(arg: i32) -> i32: - return arg + 1 +def helper(value: i32) -> i32: + return value + 1 -@export +@exported def testEntry(): logInt32(13 + 13 * helper(122)) """ From 07aeb525605fc5be7eb863559cabda27392d6373 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Mon, 5 Apr 2021 14:40:18 +0200 Subject: [PATCH 06/73] more ideas [skip-ci] --- py2wasm/python.py | 32 +++++++++++++++++++++++++++++++- py2wasm/wasm.py | 17 +++++++++++++---- tests/integration/test_simple.py | 13 +++++++++++++ 3 files changed, 57 insertions(+), 5 deletions(-) create mode 100644 tests/integration/test_simple.py diff --git a/py2wasm/python.py b/py2wasm/python.py index da7c140..853fd25 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -72,7 +72,37 @@ class Visitor: return wasm.Import(module, name, intname, _parse_import_args(node.args)) - return wasm.Function(node.name, exported) + result = _parse_annotation(node.returns) + + statements = [ + self.visit_stmt(node, stmt) + for stmt in node.body + ] + + return wasm.Function(node.name, exported, result, statements) + + def visit_stmt(self, func: ast.FunctionDef, stmt: ast.stmt) -> wasm.Statement: + """ + Visits a statement node + """ + if isinstance(stmt, ast.Return): + return self.visit_Return(func, stmt) + + raise NotImplementedError + + def visit_Return(self, func: ast.FunctionDef, stmt: ast.Return) -> wasm.Statement: + """ + Visits a statement node + """ + assert isinstance(stmt.value, ast.Constant) + assert isinstance(stmt.value.value, int) + + return_type = _parse_annotation(func.returns) + + return wasm.Statement( + '{}.const'.format(return_type), + str(stmt.value.value) + ) def _parse_import_decorator(func_name: str, args: List[ast.expr]) -> Tuple[str, str, str]: """ diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py index 0843a28..fd1b43a 100644 --- a/py2wasm/wasm.py +++ b/py2wasm/wasm.py @@ -2,7 +2,7 @@ Python classes for storing the representation of Web Assembly code """ -from typing import Iterable, List, Tuple +from typing import Iterable, List, Optional, Tuple class Import: """ @@ -49,12 +49,18 @@ class Function: """ Represents a Web Assembly function """ - def __init__(self, name: str, exported: bool = True) -> None: + def __init__( + self, + name: str, + exported: bool, + result: Optional[str], + statements: Iterable[Statement], + ) -> None: self.name = name self.exported = exported self.params: List[Tuple[str, str]] = [] - self.returns = None - self.statements: List[Statement] = [] + self.result = result + self.statements = [*statements] def generate(self) -> str: """ @@ -65,6 +71,9 @@ class Function: for nam, typ in self.params: header += ' (param ${} {})'.format(nam, typ) + if self.result: + header += ' (result {})'.format(self.result) + return '(func {}\n {})'.format( header, '\n '.join(x.generate() for x in self.statements), diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py new file mode 100644 index 0000000..99ce3ce --- /dev/null +++ b/tests/integration/test_simple.py @@ -0,0 +1,13 @@ +from .helpers import Suite + +def test_return(): + code_py = """ +@exported +def testEntry() -> i32: + return 13 +""" + + result = Suite(code_py, 'test_fib').run_code() + + assert 13 is result.returned_value + assert [] == result.log_int32_list From c234f57283f7feeb6d04a453a8eca313f7084fa2 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Mon, 5 Apr 2021 18:11:07 +0200 Subject: [PATCH 07/73] More rewriting [skip-ci] --- py2wasm/python.py | 304 +++++++++++++++++++------------ py2wasm/wasm.py | 8 +- pylintrc | 2 +- tests/integration/test_fib.py | 23 ++- tests/integration/test_simple.py | 29 ++- 5 files changed, 237 insertions(+), 129 deletions(-) diff --git a/py2wasm/python.py b/py2wasm/python.py index 853fd25..9710051 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -2,7 +2,7 @@ Code for parsing the source (python-alike) code """ -from typing import List, Optional, Union, Tuple +from typing import Dict, Generator, List, Optional, Union, Tuple import ast @@ -10,6 +10,10 @@ from . import wasm # pylint: disable=C0103,R0201 +StatementGenerator = Generator[wasm.Statement, None, None] + +WLocals = Dict[str, str] + class Visitor: """ Class to visit a Python syntax tree @@ -27,7 +31,7 @@ class Visitor: for stmt in node.body: if isinstance(stmt, ast.FunctionDef): - wnode = self.visit_FunctionDef(stmt) + wnode = self.visit_FunctionDef(module, stmt) if isinstance(wnode, wasm.Import): module.imports.append(wnode) @@ -39,13 +43,18 @@ class Visitor: return module - def visit_FunctionDef(self, node: ast.FunctionDef) -> Union[wasm.Import, wasm.Function]: + def visit_FunctionDef( + self, + module: wasm.Module, + node: ast.FunctionDef, + ) -> Union[wasm.Import, wasm.Function]: """ A Python function definition with the @external decorator is returned as import. Other functions are returned as (exported) functions. - """ + Nested / dynamicly created functions are not yet supported + """ exported = False if node.decorator_list: @@ -68,40 +77,197 @@ class Visitor: assert not call.keywords - module, name, intname = _parse_import_decorator(node.name, call.args) + mod, name, intname = _parse_import_decorator(node.name, call.args) - return wasm.Import(module, name, intname, _parse_import_args(node.args)) + return wasm.Import(mod, name, intname, _parse_import_args(node.args)) + else: + raise NotImplementedError - result = _parse_annotation(node.returns) + result = None if node.returns is None else _parse_annotation(node.returns) - statements = [ - self.visit_stmt(node, stmt) - for stmt in node.body + assert not node.args.vararg + assert not node.args.kwonlyargs + assert not node.args.kw_defaults + assert not node.args.kwarg + assert not node.args.defaults + + params = [ + (a.arg, _parse_annotation(a.annotation), ) + for a in [ + *node.args.posonlyargs, + *node.args.args, + ] ] - return wasm.Function(node.name, exported, result, statements) + wlocals: WLocals = dict(params) - def visit_stmt(self, func: ast.FunctionDef, stmt: ast.stmt) -> wasm.Statement: + statements: List[wasm.Statement] = [] + for py_stmt in node.body: + statements.extend( + self.visit_stmt(module, node, wlocals, py_stmt) + ) + + return wasm.Function(node.name, exported, params, result, statements) + + def visit_stmt( + self, + module: wasm.Module, + func: ast.FunctionDef, + wlocals: WLocals, + stmt: ast.stmt, + ) -> StatementGenerator: """ - Visits a statement node + Visits a statement node within a function """ if isinstance(stmt, ast.Return): - return self.visit_Return(func, stmt) + return self.visit_Return(module, func, wlocals, stmt) - raise NotImplementedError + if isinstance(stmt, ast.Expr): + assert isinstance(stmt.value, ast.Call) - def visit_Return(self, func: ast.FunctionDef, stmt: ast.Return) -> wasm.Statement: + return self.visit_Call(module, wlocals, "None", stmt.value) + + raise NotImplementedError(stmt) + + def visit_Return( + self, + module: wasm.Module, + func: ast.FunctionDef, + wlocals: WLocals, + stmt: ast.Return, + ) -> StatementGenerator: """ Visits a statement node """ - assert isinstance(stmt.value, ast.Constant) - assert isinstance(stmt.value.value, int) + + assert stmt.value is not None return_type = _parse_annotation(func.returns) - return wasm.Statement( - '{}.const'.format(return_type), - str(stmt.value.value) + return self.visit_expr(module, wlocals, return_type, stmt.value) + + def visit_expr( + self, + module: wasm.Module, + wlocals: WLocals, + exp_type: str, + node: ast.expr, + ) -> StatementGenerator: + """ + Visit an expression node + """ + if isinstance(node, ast.BinOp): + return self.visit_BinOp(module, wlocals, exp_type, node) + + if isinstance(node, ast.Call): + return self.visit_Call(module, wlocals, exp_type, node) + + if isinstance(node, ast.Constant): + return self.visit_Constant(exp_type, node) + + if isinstance(node, ast.Name): + return self.visit_Name(wlocals, exp_type, node) + + raise NotImplementedError(node) + + def visit_BinOp( + self, + module: wasm.Module, + wlocals: WLocals, + exp_type: str, + node: ast.BinOp, + ) -> StatementGenerator: + """ + Visits a BinOp node as (part of) an expression + """ + yield from self.visit_expr(module, wlocals, exp_type, node.left) + yield from self.visit_expr(module, wlocals, exp_type, node.right) + + if isinstance(node.op, ast.Add): + yield wasm.Statement('{}.add'.format(exp_type)) + return + + raise NotImplementedError(node.op) + + def visit_Call( + self, + module: wasm.Module, + wlocals: WLocals, + exp_type: str, + node: ast.Call, + ) -> StatementGenerator: + """ + Visits a Call node as (part of) an expression + """ + assert isinstance(node.func, ast.Name) + assert not node.keywords + + called_name = node.func.id + + called_func_list = [ + x + for x in module.functions + if x.name == called_name + ] + if called_func_list: + assert len(called_func_list) == 1 + called_params = called_func_list[0].params + called_result = called_func_list[0].result + else: + called_import_list = [ + x + for x in module.imports + if x.name == node.func.id + ] + if called_import_list: + assert len(called_import_list) == 1 + called_params = called_import_list[0].params + called_result = called_import_list[0].result + else: + assert 1 == len(called_func_list), \ + 'Could not find function {}'.format(node.func.id) + + assert exp_type == called_result + + assert len(called_params) == len(node.args), \ + '{}:{} Function {} requires {} arguments, but {} are supplied'.format( + node.lineno, node.col_offset, + called_name, len(called_params), len(node.args), + ) + + for (_, exp_expr_type), expr in zip(called_params, node.args): + yield from self.visit_expr(module, wlocals, exp_expr_type, expr) + + yield wasm.Statement( + 'call', + '${}'.format(called_name), + ) + + def visit_Constant(self, exp_type: str, node: ast.Constant) -> StatementGenerator: + """ + Visits a Constant node as (part of) an expression + """ + if 'i32' == exp_type: + assert isinstance(node.value, int) + assert -2147483648 <= node.value <= 2147483647 + + yield wasm.Statement( + '{}.const'.format(exp_type), + str(node.value) + ) + return + + raise NotImplementedError(exp_type) + + def visit_Name(self, wlocals: WLocals, exp_type: str, node: ast.Name) -> StatementGenerator: + """ + Visits a Name node as (part of) an expression + """ + assert node.id in wlocals + assert exp_type == wlocals[node.id] + yield wasm.Statement( + 'local.get', + '${}'.format(node.id), ) def _parse_import_decorator(func_name: str, args: List[ast.expr]) -> Tuple[str, str, str]: @@ -156,99 +322,3 @@ def _parse_annotation(ann: Optional[ast.expr]) -> str: result = ann.id assert result in ['i32'] return result - - - - - # def visit_ImportFrom(self, node): - # for alias in node.names: - # self.imports.append(Import( - # node.module, - # alias.name, - # alias.asname, - # )) - # - # def generic_visit(self, node: Any) -> None: - # print(node) - # super().generic_visit(node) - # - # def visit_FunctionDef(self, node: ast.FunctionDef) -> None: - # is_export = False - # - # if node.decorator_list: - # assert 1 == len(node.decorator_list) - # - # call = node.decorator_list[0] - # if not isinstance(call, ast.Name): - # assert isinstance(call, ast.Call) - # - # assert 'external' == call.func.id - # assert 1 == len(call.args) - # assert isinstance(call.args[0].value, str) - # - # import_ = Import( - # call.args[0].value, - # node.name, - # node.name, - # ) - # - # import_.params = [ - # arg.annotation.id - # for arg in node.args.args - # ] - # - # self.module.imports.append(import_) - # return - # - # assert call.id == 'export' - # is_export = True - # - # func = Function( - # node.name, - # is_export, - # ) - # - # for arg in node.args.args: - # func.params.append( - # (arg.arg, arg.annotation.id) - # ) - # func.result = node.returns - # - # self._stack.append(func) - # self.generic_visit(node) - # self._stack.pop() - # - # self.module.functions.append(func) - # - # def visit_Call(self, node: ast.Call) -> None: - # self.generic_visit(node) - # - # func = self._stack[-1] - # func.statements.append( - # Statement('call', '$' + node.func.id) - # ) - # - # def visit_BinOp(self, node: ast.BinOp) -> None: - # self.generic_visit(node) - # - # func = self._stack[-1] - # - # if 'Add' == node.op.__class__.__name__: - # func.statements.append( - # Statement('i32.add') - # ) - # elif 'Mult' == node.op.__class__.__name__: - # func.statements.append( - # Statement('i32.mul') - # ) - # else: - # raise NotImplementedError - # - # def visit_Constant(self, node: ast.Constant) -> None: - # func = self._stack[-1] - # if isinstance(node.value, int): - # func.statements.append( - # Statement('i32.const', str(node.value)) - # ) - # - # self.generic_visit(node) diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py index fd1b43a..24d1f46 100644 --- a/py2wasm/wasm.py +++ b/py2wasm/wasm.py @@ -4,6 +4,8 @@ Python classes for storing the representation of Web Assembly code from typing import Iterable, List, Optional, Tuple +Param = Tuple[str, str] + class Import: """ Represents a Web Assembly import @@ -13,12 +15,13 @@ class Import: module: str, name: str, intname: str, - params: Iterable[Tuple[str, str]], + params: Iterable[Param], ) -> None: self.module = module self.name = name self.intname = intname self.params = [*params] + self.result: str = 'None' def generate(self) -> str: """ @@ -53,12 +56,13 @@ class Function: self, name: str, exported: bool, + params: Iterable[Param], result: Optional[str], statements: Iterable[Statement], ) -> None: self.name = name self.exported = exported - self.params: List[Tuple[str, str]] = [] + self.params = [*params] self.result = result self.statements = [*statements] diff --git a/pylintrc b/pylintrc index 72d74db..eb6abcb 100644 --- a/pylintrc +++ b/pylintrc @@ -1,2 +1,2 @@ [MASTER] -disable=C0122,R0903 +disable=C0122,R0903,R0913 diff --git a/tests/integration/test_fib.py b/tests/integration/test_fib.py index fa1eaf9..ff00b05 100644 --- a/tests/integration/test_fib.py +++ b/tests/integration/test_fib.py @@ -2,19 +2,26 @@ from .helpers import Suite def test_fib(): code_py = """ -@imported('console') -def logInt32(value: i32) -> None: - ... +def helper(n: i32, a: i32, b: i32) -> i32: + if n < 1: + return a + b -def helper(value: i32) -> i32: - return value + 1 + return helper(n - 1, a + b, a) + +def fib(n: i32) -> i32: + if n == 0: + return 0 + + if n == 1: + return 1 + + return helper(n - 1, 0, 1) @exported def testEntry(): - logInt32(13 + 13 * helper(122)) + return fib(40) """ result = Suite(code_py, 'test_fib').run_code() - assert None is result.returned_value - assert [1612] == result.log_int32_list + assert 102334155 == result.returned_value diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index 99ce3ce..0d015b4 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -9,5 +9,32 @@ def testEntry() -> i32: result = Suite(code_py, 'test_fib').run_code() - assert 13 is result.returned_value + assert 13 == result.returned_value + assert [] == result.log_int32_list + +def test_addition(): + code_py = """ +@exported +def testEntry() -> i32: + return 10 + 3 +""" + + result = Suite(code_py, 'test_fib').run_code() + + assert 13 == result.returned_value + assert [] == result.log_int32_list + +def test_call(): + code_py = """ +def helper(left: i32, right: i32) -> i32: + return left + right + +@exported +def testEntry() -> i32: + return helper(10, 3) +""" + + result = Suite(code_py, 'test_fib').run_code() + + assert 13 == result.returned_value assert [] == result.log_int32_list From e972b37149d56810f7d3f42cb0ab212a69a3438d Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 7 Aug 2021 14:34:50 +0200 Subject: [PATCH 08/73] If statements \o/ --- Makefile | 2 +- README.md | 6 +++ py2wasm/python.py | 89 +++++++++++++++++++++++++++++++- requirements.txt | 1 + tests/integration/test_fib.py | 3 ++ tests/integration/test_simple.py | 69 +++++++++++++++++++++++-- 6 files changed, 164 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 3633841..9eb1785 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ server: python3.8 -m http.server test: venv/.done - venv/bin/pytest tests + venv/bin/pytest tests $(TEST_FLAGS) lint: venv/.done venv/bin/pylint py2wasm diff --git a/README.md b/README.md index 4fb6851..93683c4 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,9 @@ webasm (python) =============== You will need wat2wasm from github.com/WebAssembly/wabt in your path. + +Ideas +===== +- https://github.com/wasmerio/wasmer-python +- https://github.com/diekmann/wasm-fizzbuzz +- https://blog.scottlogic.com/2018/04/26/webassembly-by-hand.html diff --git a/py2wasm/python.py b/py2wasm/python.py index 9710051..8e2740a 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -127,6 +127,9 @@ class Visitor: return self.visit_Call(module, wlocals, "None", stmt.value) + if isinstance(stmt, ast.If): + return self.visit_If(module, func, wlocals, stmt) + raise NotImplementedError(stmt) def visit_Return( @@ -137,14 +140,44 @@ class Visitor: stmt: ast.Return, ) -> StatementGenerator: """ - Visits a statement node + Visits a Return node """ assert stmt.value is not None return_type = _parse_annotation(func.returns) - return self.visit_expr(module, wlocals, return_type, stmt.value) + yield from self.visit_expr(module, wlocals, return_type, stmt.value) + yield wasm.Statement('return') + + def visit_If( + self, + module: wasm.Module, + func: ast.FunctionDef, + wlocals: WLocals, + stmt: ast.If, + ) -> StatementGenerator: + """ + Visits an If node + """ + yield from self.visit_expr( + module, + wlocals, + 'bool', + stmt.test, + ) + + yield wasm.Statement('if') + + for py_stmt in stmt.body: + yield from self.visit_stmt(module, func, wlocals, py_stmt) + + yield wasm.Statement('else') + + for py_stmt in stmt.orelse: + yield from self.visit_stmt(module, func, wlocals, py_stmt) + + yield wasm.Statement('end') def visit_expr( self, @@ -159,6 +192,12 @@ class Visitor: if isinstance(node, ast.BinOp): return self.visit_BinOp(module, wlocals, exp_type, node) + if isinstance(node, ast.UnaryOp): + return self.visit_UnaryOp(module, wlocals, exp_type, node) + + if isinstance(node, ast.Compare): + return self.visit_Compare(module, wlocals, exp_type, node) + if isinstance(node, ast.Call): return self.visit_Call(module, wlocals, exp_type, node) @@ -170,6 +209,24 @@ class Visitor: raise NotImplementedError(node) + def visit_UnaryOp( + self, + module: wasm.Module, + wlocals: WLocals, + exp_type: str, + node: ast.UnaryOp, + ) -> StatementGenerator: + """ + Visits a UnaryOp node as (part of) an expression + """ + del module + del wlocals + + if not isinstance(node.operand, ast.Constant): + raise NotImplementedError + + return self.visit_Constant(exp_type, node.operand) + def visit_BinOp( self, module: wasm.Module, @@ -189,6 +246,34 @@ class Visitor: raise NotImplementedError(node.op) + def visit_Compare( + self, + module: wasm.Module, + wlocals: WLocals, + exp_type: str, + node: ast.Compare, + ) -> StatementGenerator: + """ + Visits a Compare node as (part of) an expression + """ + assert 'bool' == exp_type + + if 1 != len(node.ops) or 1 != len(node.comparators): + raise NotImplementedError + + yield from self.visit_expr(module, wlocals, 'i32', node.left) + yield from self.visit_expr(module, wlocals, 'i32', node.comparators[0]) + + if isinstance(node.ops[0], ast.Lt): + yield wasm.Statement('i32.lt_s') + return + + if isinstance(node.ops[0], ast.Gt): + yield wasm.Statement('i32.gt_s') + return + + raise NotImplementedError(node.ops) + def visit_Call( self, module: wasm.Module, diff --git a/requirements.txt b/requirements.txt index ef8699b..c20edb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ mypy==0.812 pylint==2.7.4 pytest==6.2.2 +pytest-integration==0.2.2 pywasm==1.0.7 diff --git a/tests/integration/test_fib.py b/tests/integration/test_fib.py index ff00b05..a68cb0e 100644 --- a/tests/integration/test_fib.py +++ b/tests/integration/test_fib.py @@ -1,5 +1,8 @@ +import pytest + from .helpers import Suite +@pytest.mark.slow_integration_test def test_fib(): code_py = """ def helper(n: i32, a: i32, b: i32) -> i32: diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index 0d015b4..fdcab75 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -1,5 +1,8 @@ +import pytest + from .helpers import Suite +@pytest.mark.integration_test def test_return(): code_py = """ @exported @@ -7,11 +10,25 @@ def testEntry() -> i32: return 13 """ - result = Suite(code_py, 'test_fib').run_code() + result = Suite(code_py, 'test_return').run_code() assert 13 == result.returned_value assert [] == result.log_int32_list +@pytest.mark.integration_test +def test_arg(): + code_py = """ +@exported +def testEntry(a: i32) -> i32: + return a +""" + + result = Suite(code_py, 'test_return').run_code(125) + + assert 125 == result.returned_value + assert [] == result.log_int32_list + +@pytest.mark.integration_test def test_addition(): code_py = """ @exported @@ -19,11 +36,57 @@ def testEntry() -> i32: return 10 + 3 """ - result = Suite(code_py, 'test_fib').run_code() + result = Suite(code_py, 'test_addition').run_code() assert 13 == result.returned_value assert [] == result.log_int32_list +@pytest.mark.integration_test +def test_if_simple(): + code_py = """ +@exported +def testEntry(a: i32) -> i32: + if a > 10: + return 1 + + return 0 +""" + + suite = Suite(code_py, 'test_return') + + result = suite.run_code(10) + assert 0 == result.returned_value + assert [] == result.log_int32_list + + result = suite.run_code(11) + assert 1 == result.returned_value + +@pytest.mark.integration_test +def test_if_complex(): + code_py = """ +@exported +def testEntry(a: i32) -> i32: + if a > 10: + return 10 + elif a > 0: + return a + else: + return 0 + + return -1 # Required due to function type +""" + + suite = Suite(code_py, 'test_return') + + assert 10 == suite.run_code(20).returned_value + assert 10 == suite.run_code(10).returned_value + + assert 8 == suite.run_code(8).returned_value + + assert 0 == suite.run_code(0).returned_value + assert 0 == suite.run_code(-1).returned_value + +@pytest.mark.integration_test def test_call(): code_py = """ def helper(left: i32, right: i32) -> i32: @@ -34,7 +97,7 @@ def testEntry() -> i32: return helper(10, 3) """ - result = Suite(code_py, 'test_fib').run_code() + result = Suite(code_py, 'test_call').run_code() assert 13 == result.returned_value assert [] == result.log_int32_list From 0c64973b2b3b4f3353a5a8305d82cc643be1ea67 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 7 Aug 2021 14:40:15 +0200 Subject: [PATCH 09/73] UAdd, tests --- py2wasm/python.py | 16 +++++++++++----- tests/integration/test_simple.py | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/py2wasm/python.py b/py2wasm/python.py index 8e2740a..90f97af 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -219,13 +219,19 @@ class Visitor: """ Visits a UnaryOp node as (part of) an expression """ - del module - del wlocals + if isinstance(node.op, ast.UAdd): + return self.visit_expr(module, wlocals, exp_type, node.operand) - if not isinstance(node.operand, ast.Constant): - raise NotImplementedError + if isinstance(node.op, ast.USub): + if not isinstance(node.operand, ast.Constant): + raise NotImplementedError(node.operand) - return self.visit_Constant(exp_type, node.operand) + if not isinstance(node.operand.value, int): + raise NotImplementedError(node.operand.value) + + return self.visit_Constant(exp_type, ast.Constant(-node.operand.value)) + + raise NotImplementedError(node.op) def visit_BinOp( self, diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index fdcab75..17b2b86 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -28,6 +28,32 @@ def testEntry(a: i32) -> i32: assert 125 == result.returned_value assert [] == result.log_int32_list +@pytest.mark.integration_test +def test_uadd(): + code_py = """ +@exported +def testEntry() -> i32: + return +523 +""" + + result = Suite(code_py, 'test_addition').run_code() + + assert 523 == result.returned_value + assert [] == result.log_int32_list + +@pytest.mark.integration_test +def test_usub(): + code_py = """ +@exported +def testEntry() -> i32: + return -19 +""" + + result = Suite(code_py, 'test_addition').run_code() + + assert -19 == result.returned_value + assert [] == result.log_int32_list + @pytest.mark.integration_test def test_addition(): code_py = """ From b26efc797d70b7e3fb0986faa941acbf6511ad80 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 7 Aug 2021 14:52:51 +0200 Subject: [PATCH 10/73] Cleanup, added tests --- py2wasm/python.py | 31 +++++++++++++------------------ tests/integration/test_simple.py | 18 +++++++++++++++++- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/py2wasm/python.py b/py2wasm/python.py index 90f97af..9c6e989 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -295,28 +295,23 @@ class Visitor: called_name = node.func.id + search_list: List[Union[wasm.Function, wasm.Import]] + search_list = [ + *module.functions, + *module.imports, + ] + called_func_list = [ x - for x in module.functions + for x in search_list if x.name == called_name ] - if called_func_list: - assert len(called_func_list) == 1 - called_params = called_func_list[0].params - called_result = called_func_list[0].result - else: - called_import_list = [ - x - for x in module.imports - if x.name == node.func.id - ] - if called_import_list: - assert len(called_import_list) == 1 - called_params = called_import_list[0].params - called_result = called_import_list[0].result - else: - assert 1 == len(called_func_list), \ - 'Could not find function {}'.format(node.func.id) + + assert 1 == len(called_func_list), \ + 'Could not find function {}'.format(node.func.id) + + called_params = called_func_list[0].params + called_result = called_func_list[0].result assert exp_type == called_result diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index 17b2b86..af73c4d 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -113,7 +113,7 @@ def testEntry(a: i32) -> i32: assert 0 == suite.run_code(-1).returned_value @pytest.mark.integration_test -def test_call(): +def test_call_pre_defined(): code_py = """ def helper(left: i32, right: i32) -> i32: return left + right @@ -127,3 +127,19 @@ def testEntry() -> i32: assert 13 == result.returned_value assert [] == result.log_int32_list + +@pytest.mark.integration_test +def test_call_post_defined(): + code_py = """ +@exported +def testEntry() -> i32: + return helper(10, 3) + +def helper(left: i32, right: i32) -> i32: + return left - right +""" + + result = Suite(code_py, 'test_call').run_code() + + assert 7 == result.returned_value + assert [] == result.log_int32_list From 9616d2046073c62692cc9ba389bccb34786c8f7c Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 7 Aug 2021 15:02:20 +0200 Subject: [PATCH 11/73] fib works \o/ --- py2wasm/python.py | 44 +++++++++++++++++++++++++++++++---- tests/integration/test_fib.py | 2 +- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/py2wasm/python.py b/py2wasm/python.py index 9c6e989..57d004e 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -29,21 +29,36 @@ class Visitor: module = wasm.Module() + # Do a check first for all function definitions + # to get their types. Otherwise you cannot call + # a method that you haven't defined just yet, + # even if it is in the same file + function_body_map: Dict[ast.FunctionDef, wasm.Function] = {} for stmt in node.body: if isinstance(stmt, ast.FunctionDef): - wnode = self.visit_FunctionDef(module, stmt) + wnode = self.pre_visit_FunctionDef(module, stmt) if isinstance(wnode, wasm.Import): module.imports.append(wnode) else: module.functions.append(wnode) + function_body_map[stmt] = wnode + continue + + # No other pre visits to do + + for stmt in node.body: + if isinstance(stmt, ast.FunctionDef): + if stmt in function_body_map: + self.parse_FunctionDef_body(module, function_body_map[stmt], stmt) + # else: It's an import, no actual body to parse continue raise NotImplementedError(stmt) return module - def visit_FunctionDef( + def pre_visit_FunctionDef( self, module: wasm.Module, node: ast.FunctionDef, @@ -55,6 +70,8 @@ class Visitor: Nested / dynamicly created functions are not yet supported """ + del module + exported = False if node.decorator_list: @@ -99,7 +116,18 @@ class Visitor: ] ] - wlocals: WLocals = dict(params) + return wasm.Function(node.name, exported, params, result, []) + + def parse_FunctionDef_body( + self, + module: wasm.Module, + func: wasm.Function, + node: ast.FunctionDef, + ) -> None: + """ + Parses the function body + """ + wlocals: WLocals = dict(func.params) statements: List[wasm.Statement] = [] for py_stmt in node.body: @@ -107,7 +135,7 @@ class Visitor: self.visit_stmt(module, node, wlocals, py_stmt) ) - return wasm.Function(node.name, exported, params, result, statements) + func.statements = statements def visit_stmt( self, @@ -250,6 +278,10 @@ class Visitor: yield wasm.Statement('{}.add'.format(exp_type)) return + if isinstance(node.op, ast.Sub): + yield wasm.Statement('{}.sub'.format(exp_type)) + return + raise NotImplementedError(node.op) def visit_Compare( @@ -274,6 +306,10 @@ class Visitor: yield wasm.Statement('i32.lt_s') return + if isinstance(node.ops[0], ast.Eq): + yield wasm.Statement('i32.eq') + return + if isinstance(node.ops[0], ast.Gt): yield wasm.Statement('i32.gt_s') return diff --git a/tests/integration/test_fib.py b/tests/integration/test_fib.py index a68cb0e..b27297d 100644 --- a/tests/integration/test_fib.py +++ b/tests/integration/test_fib.py @@ -21,7 +21,7 @@ def fib(n: i32) -> i32: return helper(n - 1, 0, 1) @exported -def testEntry(): +def testEntry() -> i32: return fib(40) """ From 98d3d8848a659739eb254dd81a6a305a8c232868 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 7 Aug 2021 15:24:10 +0200 Subject: [PATCH 12/73] i64, f32, f64 (some conversions) --- py2wasm/python.py | 39 ++++++++++++-- tests/integration/test_simple.py | 93 +++++++++++++++++++++++++++++++- 2 files changed, 126 insertions(+), 6 deletions(-) diff --git a/py2wasm/python.py b/py2wasm/python.py index 57d004e..77b53d2 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -373,10 +373,28 @@ class Visitor: assert isinstance(node.value, int) assert -2147483648 <= node.value <= 2147483647 - yield wasm.Statement( - '{}.const'.format(exp_type), - str(node.value) - ) + yield wasm.Statement('i32.const', str(node.value)) + return + + if 'i64' == exp_type: + assert isinstance(node.value, int) + assert -9223372036854775808 <= node.value <= 9223372036854775807 + + yield wasm.Statement('i64.const', str(node.value)) + return + + if 'f32' == exp_type: + assert isinstance(node.value, float) + # TODO: Size check? + + yield wasm.Statement('f32.const', node.value.hex()) + return + + if 'f64' == exp_type: + assert isinstance(node.value, float) + # TODO: Size check? + + yield wasm.Statement('f64.const', node.value.hex()) return raise NotImplementedError(exp_type) @@ -386,6 +404,17 @@ class Visitor: Visits a Name node as (part of) an expression """ assert node.id in wlocals + + if exp_type == 'i64' and wlocals[node.id] == 'i32': + yield wasm.Statement('local.get', '${}'.format(node.id)) + yield wasm.Statement('i64.extend_i32_s') + return + + if exp_type == 'f64' and wlocals[node.id] == 'f32': + yield wasm.Statement('local.get', '${}'.format(node.id)) + yield wasm.Statement('f64.promote_f32') + return + assert exp_type == wlocals[node.id] yield wasm.Statement( 'local.get', @@ -442,5 +471,5 @@ def _parse_annotation(ann: Optional[ast.expr]) -> str: assert ann is not None, 'Web Assembly requires type annotations' assert isinstance(ann, ast.Name) result = ann.id - assert result in ['i32'] + assert result in ['i32', 'i64', 'f32', 'f64'] return result diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index af73c4d..7650c6c 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -3,7 +3,7 @@ import pytest from .helpers import Suite @pytest.mark.integration_test -def test_return(): +def test_return_i32(): code_py = """ @exported def testEntry() -> i32: @@ -15,6 +15,45 @@ def testEntry() -> i32: assert 13 == result.returned_value assert [] == result.log_int32_list +@pytest.mark.integration_test +def test_return_i64(): + code_py = """ +@exported +def testEntry() -> i64: + return 13 +""" + + result = Suite(code_py, 'test_return').run_code() + + assert 13 == result.returned_value + assert [] == result.log_int32_list + +@pytest.mark.integration_test +def test_return_f32(): + code_py = """ +@exported +def testEntry() -> f32: + return 13.5 +""" + + result = Suite(code_py, 'test_return').run_code() + + assert 13.5 == result.returned_value + assert [] == result.log_int32_list + +@pytest.mark.integration_test +def test_return_f64(): + code_py = """ +@exported +def testEntry() -> f64: + return 13.5 +""" + + result = Suite(code_py, 'test_return').run_code() + + assert 13.5 == result.returned_value + assert [] == result.log_int32_list + @pytest.mark.integration_test def test_arg(): code_py = """ @@ -28,6 +67,58 @@ def testEntry(a: i32) -> i32: assert 125 == result.returned_value assert [] == result.log_int32_list +@pytest.mark.integration_test +def test_i32_to_i64(): + code_py = """ +@exported +def testEntry(a: i32) -> i64: + return a +""" + + result = Suite(code_py, 'test_return').run_code(125) + + assert 125 == result.returned_value + assert [] == result.log_int32_list + +@pytest.mark.integration_test +def test_i32_plus_i64(): + code_py = """ +@exported +def testEntry(a: i32, b: i64) -> i64: + return a + b +""" + + result = Suite(code_py, 'test_return').run_code(125, 100) + + assert 225 == result.returned_value + assert [] == result.log_int32_list + +@pytest.mark.integration_test +def test_f32_to_f64(): + code_py = """ +@exported +def testEntry(a: f32) -> f64: + return a +""" + + result = Suite(code_py, 'test_return').run_code(125.5) + + assert 125.5 == result.returned_value + assert [] == result.log_int32_list + +@pytest.mark.integration_test +def test_f32_plus_f64(): + code_py = """ +@exported +def testEntry(a: f32, b: f64) -> f64: + return a + b +""" + + result = Suite(code_py, 'test_return').run_code(125.5, 100.25) + + assert 225.75 == result.returned_value + assert [] == result.log_int32_list + @pytest.mark.integration_test def test_uadd(): code_py = """ From 3aef459924ab026f08a73f6a50c3cab9a09908aa Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Fri, 4 Mar 2022 09:57:35 +0100 Subject: [PATCH 13/73] idea / scaffolding [skip-ci] --- py2wasm/python.py | 97 +++++++++++++++++++++++++++----- py2wasm/wasm.py | 27 ++++++++- tests/integration/test_fib.py | 1 + tests/integration/test_simple.py | 37 ++++++++++++ 4 files changed, 148 insertions(+), 14 deletions(-) diff --git a/py2wasm/python.py b/py2wasm/python.py index 77b53d2..6f5d36b 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -45,6 +45,11 @@ class Visitor: function_body_map[stmt] = wnode continue + if isinstance(stmt, ast.ClassDef): + wclass = self.pre_visit_ClassDef(module, stmt) + module.classes.append(wclass) + continue + # No other pre visits to do for stmt in node.body: @@ -54,6 +59,9 @@ class Visitor: # else: It's an import, no actual body to parse continue + if isinstance(stmt, ast.ClassDef): + continue + raise NotImplementedError(stmt) return module @@ -70,8 +78,6 @@ class Visitor: Nested / dynamicly created functions are not yet supported """ - del module - exported = False if node.decorator_list: @@ -108,13 +114,22 @@ class Visitor: assert not node.args.kwarg assert not node.args.defaults - params = [ - (a.arg, _parse_annotation(a.annotation), ) - for a in [ - *node.args.posonlyargs, - *node.args.args, - ] - ] + class_lookup = { + x.name: x + for x in module.classes + } + + params = [] + for arg in [*node.args.posonlyargs, *node.args.args]: + if not isinstance(arg.annotation, ast.Name): + raise NotImplementedError + + print(class_lookup) + print(arg.annotation.id) + if arg.annotation.id in class_lookup: + params.append((arg.arg, arg.annotation.id, )) + else: + params.append((arg.arg, _parse_annotation(arg.annotation), )) return wasm.Function(node.name, exported, params, result, []) @@ -137,6 +152,51 @@ class Visitor: func.statements = statements + def pre_visit_ClassDef( + self, + module: wasm.Module, + node: ast.ClassDef, + ) -> wasm.Class: + """ + TODO: Document this + """ + del module + + if node.bases or node.keywords or node.decorator_list: + raise NotImplementedError + + members: List[wasm.ClassMember] = [] + + for stmt in node.body: + if not isinstance(stmt, ast.AnnAssign): + raise NotImplementedError + + if not isinstance(stmt.target, ast.Name): + raise NotImplementedError + + if not isinstance(stmt.annotation, ast.Name): + raise NotImplementedError + + if stmt.annotation.id != 'i32': + raise NotImplementedError + + if stmt.value is None: + default = None + else: + if not isinstance(stmt.value, ast.Constant): + raise NotImplementedError + + if not isinstance(stmt.value.value, int): + raise NotImplementedError + + default = wasm.Constant(stmt.value.value) + + members.append(wasm.ClassMember( + stmt.target.id, stmt.annotation.id, default + )) + + return wasm.Class(node.name, members) + def visit_stmt( self, module: wasm.Module, @@ -235,6 +295,9 @@ class Visitor: if isinstance(node, ast.Name): return self.visit_Name(wlocals, exp_type, node) + if isinstance(node, ast.Attribute): + return [] # TODO + raise NotImplementedError(node) def visit_UnaryOp( @@ -331,10 +394,11 @@ class Visitor: called_name = node.func.id - search_list: List[Union[wasm.Function, wasm.Import]] + search_list: List[Union[wasm.Function, wasm.Import, wasm.Class]] search_list = [ *module.functions, *module.imports, + *module.classes, ] called_func_list = [ @@ -346,8 +410,15 @@ class Visitor: assert 1 == len(called_func_list), \ 'Could not find function {}'.format(node.func.id) - called_params = called_func_list[0].params - called_result = called_func_list[0].result + if isinstance(called_func_list[0], wasm.Class): + called_params = [ + (x.name, x.type, ) + for x in called_func_list[0].members + ] + called_result: Optional[str] = called_func_list[0].name + else: + called_params = called_func_list[0].params + called_result = called_func_list[0].result assert exp_type == called_result @@ -471,5 +542,5 @@ def _parse_annotation(ann: Optional[ast.expr]) -> str: assert ann is not None, 'Web Assembly requires type annotations' assert isinstance(ann, ast.Name) result = ann.id - assert result in ['i32', 'i64', 'f32', 'f64'] + assert result in ['i32', 'i64', 'f32', 'f64'], result return result diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py index 24d1f46..68300fc 100644 --- a/py2wasm/wasm.py +++ b/py2wasm/wasm.py @@ -2,7 +2,7 @@ Python classes for storing the representation of Web Assembly code """ -from typing import Iterable, List, Optional, Tuple +from typing import Iterable, List, Optional, Tuple, Union Param = Tuple[str, str] @@ -83,6 +83,30 @@ class Function: '\n '.join(x.generate() for x in self.statements), ) +class Constant: + """ + TODO + """ + def __init__(self, value: Union[None, bool, int, float]) -> None: + self.value = value + +class ClassMember: + """ + Represents a Web Assembly class member + """ + def __init__(self, name: str, type_: str, default: Optional[Constant]) -> None: + self.name = name + self.type = type_ + self.default = default + +class Class: + """ + Represents a Web Assembly class + """ + def __init__(self, name: str, members: List[ClassMember]) -> None: + self.name = name + self.members = members + class Module: """ Represents a Web Assembly module @@ -90,6 +114,7 @@ class Module: def __init__(self) -> None: self.imports: List[Import] = [] self.functions: List[Function] = [] + self.classes: List[Class] = [] def generate(self) -> str: """ diff --git a/tests/integration/test_fib.py b/tests/integration/test_fib.py index b27297d..bd701c9 100644 --- a/tests/integration/test_fib.py +++ b/tests/integration/test_fib.py @@ -28,3 +28,4 @@ def testEntry() -> i32: result = Suite(code_py, 'test_fib').run_code() assert 102334155 == result.returned_value + assert False diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index 7650c6c..c399e86 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -234,3 +234,40 @@ def helper(left: i32, right: i32) -> i32: assert 7 == result.returned_value assert [] == result.log_int32_list + +@pytest.mark.integration_test +def test_assign(): + code_py = """ + +@exported +def testEntry() -> i32: + a: i32 = 8947 + return a +""" + + result = Suite(code_py, 'test_call').run_code() + + assert 8947 == result.returned_value + assert [] == result.log_int32_list + +@pytest.mark.integration_test +def test_struct(): + code_py = """ + +class Rectangle: + height: i32 + width: i32 + border: i32 # = 5 + +@exported +def testEntry() -> i32: + return helper(Rectangle(100, 150, 2)) + +def helper(shape: Rectangle) -> i32: + return shape.height + shape.width + shape.border +""" + + result = Suite(code_py, 'test_call').run_code() + + assert 100 == result.returned_value + assert [] == result.log_int32_list From 541b3a3b620e982ef80f5364376fd16983f8e7b5 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Fri, 4 Mar 2022 12:48:43 +0100 Subject: [PATCH 14/73] Learning about data usage [skip-ci] --- Makefile | 13 ++++++++++++- tests/integration/test_helper.py | 33 ++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 tests/integration/test_helper.py diff --git a/Makefile b/Makefile index 9eb1785..eef7502 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,19 @@ +WABT_DIR := /home/johan/Sources/github.com/WebAssembly/wabt + +WAT2WASM := $(WABT_DIR)/bin/wat2wasm +WASM2C := $(WABT_DIR)/bin/wasm2c + %.wat: %.py py2wasm/*.py python3.8 -m py2wasm $< $@ %.wasm: %.wat - wat2wasm $^ -o $@ + $(WAT2WASM) $^ -o $@ + +%.c: %.wasm + $(WASM2C) $^ -o $@ + +# %.exe: %.c +# cc $^ -o $@ -I $(WABT_DIR)/wasm2c server: python3.8 -m http.server diff --git a/tests/integration/test_helper.py b/tests/integration/test_helper.py new file mode 100644 index 0000000..cde6560 --- /dev/null +++ b/tests/integration/test_helper.py @@ -0,0 +1,33 @@ +import io + +import pytest + +from pywasm import binary +from pywasm import Runtime + +from .helpers import wat2wasm + +@pytest.mark.parametrize('size,offset,output', [ + ('32', 0, 0x3020100), + ('32', 1, 0x4030201), + ('64', 0, 0x706050403020100), + ('64', 2, 0x908070605040302), +]) +def test_i32_64_load(size, offset, output): + code_wat = f""" + (module + (memory 1) + (data (memory 0) (i32.const 0) "\\00\\01\\02\\03\\04\\05\\06\\07\\08\\09\\10") + + (func (export "testEntry") (result i{size}) + i32.const {offset} + i{size}.load + return )) +""" + code_wasm = wat2wasm(code_wat) + module = binary.Module.from_reader(io.BytesIO(code_wasm)) + + runtime = Runtime(module, {}, {}) + + out_put = runtime.exec('testEntry', []) + assert output == out_put From 6932741ac577322ea8bad5a912c8754f50863541 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Fri, 4 Mar 2022 13:18:32 +0100 Subject: [PATCH 15/73] Learning about stacks and locals [skip-ci] --- tests/integration/test_helper.py | 42 +++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_helper.py b/tests/integration/test_helper.py index cde6560..7b2226e 100644 --- a/tests/integration/test_helper.py +++ b/tests/integration/test_helper.py @@ -7,13 +7,13 @@ from pywasm import Runtime from .helpers import wat2wasm -@pytest.mark.parametrize('size,offset,output', [ +@pytest.mark.parametrize('size,offset,exp_out_put', [ ('32', 0, 0x3020100), ('32', 1, 0x4030201), ('64', 0, 0x706050403020100), ('64', 2, 0x908070605040302), ]) -def test_i32_64_load(size, offset, output): +def test_i32_64_load(size, offset, exp_out_put): code_wat = f""" (module (memory 1) @@ -30,4 +30,40 @@ def test_i32_64_load(size, offset, output): runtime = Runtime(module, {}, {}) out_put = runtime.exec('testEntry', []) - assert output == out_put + assert exp_out_put == out_put + +def test_load_then_store(): + code_wat = """ + (module + (memory 1) + (data (memory 0) (i32.const 0) "\\04\\00\\00\\00") + + (func (export "testEntry") (result i32) (local $my_memory_value i32) + ;; Load i32 from address 0 + i32.const 0 + i32.load + + ;; Add 8 to the loaded value + i32.const 8 + i32.add + + local.set $my_memory_value + + ;; Store back to the memory + i32.const 0 + local.get $my_memory_value + i32.store + + ;; Return something + i32.const 9 + return )) +""" + code_wasm = wat2wasm(code_wat) + module = binary.Module.from_reader(io.BytesIO(code_wasm)) + + runtime = Runtime(module, {}, {}) + + out_put = runtime.exec('testEntry', []) + assert 9 == out_put + + assert (b'\x0c'+ b'\00' * 23) == runtime.store.mems[0].data[:24] From 1a35710da42fda776ebdc8c32cdc91ef0f764c3a Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Fri, 4 Mar 2022 13:27:38 +0100 Subject: [PATCH 16/73] cleanup [skip-ci] --- tests/integration/test_helper.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_helper.py b/tests/integration/test_helper.py index 7b2226e..387eb5a 100644 --- a/tests/integration/test_helper.py +++ b/tests/integration/test_helper.py @@ -7,6 +7,15 @@ from pywasm import Runtime from .helpers import wat2wasm +def run(code_wat): + code_wasm = wat2wasm(code_wat) + module = binary.Module.from_reader(io.BytesIO(code_wasm)) + + runtime = Runtime(module, {}, {}) + + out_put = runtime.exec('testEntry', []) + return (runtime, out_put) + @pytest.mark.parametrize('size,offset,exp_out_put', [ ('32', 0, 0x3020100), ('32', 1, 0x4030201), @@ -24,12 +33,8 @@ def test_i32_64_load(size, offset, exp_out_put): i{size}.load return )) """ - code_wasm = wat2wasm(code_wat) - module = binary.Module.from_reader(io.BytesIO(code_wasm)) - runtime = Runtime(module, {}, {}) - - out_put = runtime.exec('testEntry', []) + (_, out_put) = run(code_wat) assert exp_out_put == out_put def test_load_then_store(): @@ -58,12 +63,8 @@ def test_load_then_store(): i32.const 9 return )) """ - code_wasm = wat2wasm(code_wat) - module = binary.Module.from_reader(io.BytesIO(code_wasm)) + (runtime, out_put) = run(code_wat) - runtime = Runtime(module, {}, {}) - - out_put = runtime.exec('testEntry', []) assert 9 == out_put assert (b'\x0c'+ b'\00' * 23) == runtime.store.mems[0].data[:24] From bd7e8d33bf7066ad4b415e17cb617194388f243d Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Fri, 4 Mar 2022 13:35:08 +0100 Subject: [PATCH 17/73] Partial implementation [skip-ci] --- py2wasm/python.py | 68 ++++++++++++++++++++++++++++++++++++++--------- py2wasm/wasm.py | 3 ++- 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/py2wasm/python.py b/py2wasm/python.py index 6f5d36b..96062e7 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -124,10 +124,8 @@ class Visitor: if not isinstance(arg.annotation, ast.Name): raise NotImplementedError - print(class_lookup) - print(arg.annotation.id) if arg.annotation.id in class_lookup: - params.append((arg.arg, arg.annotation.id, )) + params.append((arg.arg, 'i32', )) else: params.append((arg.arg, _parse_annotation(arg.annotation), )) @@ -296,7 +294,7 @@ class Visitor: return self.visit_Name(wlocals, exp_type, node) if isinstance(node, ast.Attribute): - return [] # TODO + return self.visit_Attribute(wlocals, exp_type, node) raise NotImplementedError(node) @@ -411,14 +409,46 @@ class Visitor: 'Could not find function {}'.format(node.func.id) if isinstance(called_func_list[0], wasm.Class): - called_params = [ - (x.name, x.type, ) - for x in called_func_list[0].members - ] - called_result: Optional[str] = called_func_list[0].name - else: - called_params = called_func_list[0].params - called_result = called_func_list[0].result + return self.visit_Call_class(module, wlocals, exp_type, node, called_func_list[0]) + + return self.visit_Call_func(module, wlocals, exp_type, node, called_func_list[0]) + + def visit_Call_class( + self, + module: wasm.Module, + wlocals: WLocals, + exp_type: str, + node: ast.Call, + func: wasm.Class, + ) -> StatementGenerator: + """ + Visits a Call node as (part of) an expression + + This instantiates the class + """ + + # TODO: malloc call + + yield wasm.Statement('i32.const 0') + yield wasm.Statement('i32.load') + + + def visit_Call_func( + self, + module: wasm.Module, + wlocals: WLocals, + exp_type: str, + node: ast.Call, + func: Union[wasm.Function, wasm.Import], + ) -> StatementGenerator: + """ + Visits a Call node as (part of) an expression + """ + assert isinstance(node.func, ast.Name) + + called_name = node.func.id + called_params = func.params + called_result = func.result assert exp_type == called_result @@ -470,6 +500,20 @@ class Visitor: raise NotImplementedError(exp_type) + def visit_Attribute(self, wlocals: WLocals, exp_type: str, node: ast.Attribute) -> StatementGenerator: + """ + Visits an Attribute node as (part of) an expression + """ + + if not isinstance(node.value, ast.Name): + raise NotImplementedError + + if not isinstance(node.ctx, ast.Load): + raise NotImplementedError + + yield wasm.Statement('local.get', '$' + node.value.id) + yield wasm.Statement(exp_type + '.load', 'offset=0') # TODO: Calculate offset based on set struct + def visit_Name(self, wlocals: WLocals, exp_type: str, node: ast.Name) -> StatementGenerator: """ Visits a Name node as (part of) an expression diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py index 68300fc..70694bd 100644 --- a/py2wasm/wasm.py +++ b/py2wasm/wasm.py @@ -120,7 +120,8 @@ class Module: """ Generates the text version """ - return '(module\n {}\n {})\n'.format( + return '(module\n (memory 1)\n (data (memory 0) (i32.const 0) {})\n {}\n {})\n'.format( + '"\\\\04\\\\00\\\\00\\\\00"', '\n '.join(x.generate() for x in self.imports), '\n '.join(x.generate() for x in self.functions), ) From b468ffa780a1fefefba6e402f3f8c1a0b4802c77 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Fri, 4 Mar 2022 15:50:53 +0100 Subject: [PATCH 18/73] First memory test checks out [skip-ci] --- py2wasm/python.py | 69 +++++++++++++++++++++++++++----- py2wasm/wasm.py | 28 ++++++++++--- tests/integration/helpers.py | 32 +++++++++++++++ tests/integration/test_fib.py | 1 - tests/integration/test_simple.py | 3 +- 5 files changed, 117 insertions(+), 16 deletions(-) diff --git a/py2wasm/python.py b/py2wasm/python.py index 96062e7..a0f6e6c 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -29,6 +29,18 @@ class Visitor: module = wasm.Module() + module.functions.append(wasm.Function( + '___new_reference___', + False, + [ + ('alloc_size', 'i32'), + ], + 'i32', + [ + wasm.Statement('i32.const', '4', comment='Stub') + ] + )) + # Do a check first for all function definitions # to get their types. Otherwise you cannot call # a method that you haven't defined just yet, @@ -165,6 +177,7 @@ class Visitor: members: List[wasm.ClassMember] = [] + offset = 0 for stmt in node.body: if not isinstance(stmt, ast.AnnAssign): raise NotImplementedError @@ -189,9 +202,12 @@ class Visitor: default = wasm.Constant(stmt.value.value) - members.append(wasm.ClassMember( - stmt.target.id, stmt.annotation.id, default - )) + member = wasm.ClassMember( + stmt.target.id, stmt.annotation.id, offset, default + ) + + members.append(member) + offset += member.alloc_size() return wasm.Class(node.name, members) @@ -294,7 +310,7 @@ class Visitor: return self.visit_Name(wlocals, exp_type, node) if isinstance(node, ast.Attribute): - return self.visit_Attribute(wlocals, exp_type, node) + return self.visit_Attribute(module, wlocals, exp_type, node) raise NotImplementedError(node) @@ -419,7 +435,7 @@ class Visitor: wlocals: WLocals, exp_type: str, node: ast.Call, - func: wasm.Class, + cls: wasm.Class, ) -> StatementGenerator: """ Visits a Call node as (part of) an expression @@ -429,9 +445,20 @@ class Visitor: # TODO: malloc call - yield wasm.Statement('i32.const 0') - yield wasm.Statement('i32.load') + yield wasm.Statement('i32.const', str(cls.alloc_size())) + yield wasm.Statement('call', '$___new_reference___') + yield wasm.Statement('local.set', '$___new_reference___addr') + + for member, arg in zip(cls.members, node.args): + if not isinstance(arg, ast.Constant): + raise NotImplementedError + + yield wasm.Statement('local.get', '$___new_reference___addr') + yield wasm.Statement(f'{member.type}.const', str(arg.value)) + yield wasm.Statement(f'{member.type}.store', 'offset=' + str(member.offset)) + + yield wasm.Statement('local.get', '$___new_reference___addr') def visit_Call_func( self, @@ -500,7 +527,13 @@ class Visitor: raise NotImplementedError(exp_type) - def visit_Attribute(self, wlocals: WLocals, exp_type: str, node: ast.Attribute) -> StatementGenerator: + def visit_Attribute( + self, + module: wasm.Module, + wlocals: WLocals, + exp_type: str, + node: ast.Attribute, + ) -> StatementGenerator: """ Visits an Attribute node as (part of) an expression """ @@ -511,8 +544,26 @@ class Visitor: if not isinstance(node.ctx, ast.Load): raise NotImplementedError + cls_list = [ + x + for x in module.classes + if x.name == 'Rectangle' # TODO: STUB, since we can't acces the type properly + ] + + assert len(cls_list) == 1 + cls = cls_list[0] + + member_list = [ + x + for x in cls.members + if x.name == node.attr + ] + + assert len(member_list) == 1 + member = member_list[0] + yield wasm.Statement('local.get', '$' + node.value.id) - yield wasm.Statement(exp_type + '.load', 'offset=0') # TODO: Calculate offset based on set struct + yield wasm.Statement(exp_type + '.load', 'offset=' + str(member.offset)) def visit_Name(self, wlocals: WLocals, exp_type: str, node: ast.Name) -> StatementGenerator: """ diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py index 70694bd..338e5a0 100644 --- a/py2wasm/wasm.py +++ b/py2wasm/wasm.py @@ -38,15 +38,19 @@ class Statement: """ Represents a Web Assembly statement """ - def __init__(self, name: str, *args: str): + def __init__(self, name: str, *args: str, comment: Optional[str] = None): self.name = name self.args = args + self.comment = comment def generate(self) -> str: """ Generates the text version """ - return '{} {}'.format(self.name, ' '.join(self.args)) + args = ' '.join(self.args) + comment = f' ;; {self.comment}' if self.comment else '' + + return f'{self.name} {args}{comment}' class Function: """ @@ -78,7 +82,9 @@ class Function: if self.result: header += ' (result {})'.format(self.result) - return '(func {}\n {})'.format( + header += ' (local $___new_reference___addr i32)' + + return '(func {}\n {}\n )'.format( header, '\n '.join(x.generate() for x in self.statements), ) @@ -94,11 +100,20 @@ class ClassMember: """ Represents a Web Assembly class member """ - def __init__(self, name: str, type_: str, default: Optional[Constant]) -> None: + def __init__(self, name: str, type_: str, offset: int, default: Optional[Constant]) -> None: self.name = name self.type = type_ + self.offset = offset self.default = default + def alloc_size(self) -> int: + SIZE_MAP = { + 'i32': 4, + 'i64': 4, + } + + return SIZE_MAP[self.type] + class Class: """ Represents a Web Assembly class @@ -107,6 +122,9 @@ class Class: self.name = name self.members = members + def alloc_size(self) -> int: + return sum(x.alloc_size() for x in self.members) + class Module: """ Represents a Web Assembly module @@ -121,7 +139,7 @@ class Module: Generates the text version """ return '(module\n (memory 1)\n (data (memory 0) (i32.const 0) {})\n {}\n {})\n'.format( - '"\\\\04\\\\00\\\\00\\\\00"', + '"\\04\\00\\00\\00"', '\n '.join(x.generate() for x in self.imports), '\n '.join(x.generate() for x in self.functions), ) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 57ed359..444e1c0 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -31,6 +31,7 @@ def wat2wasm(code_wat): class SuiteResult: def __init__(self): self.log_int32_list = [] + self.returned_value = None def callback_log_int32(self, store, value): del store # auto passed by pywasm @@ -58,6 +59,10 @@ class Suite: """ code_wat = process(self.code_py, self.test_name) + dashes = '-' * 16 + + sys.stderr.write(f'{dashes} Assembly {dashes}\n') + line_list = code_wat.split('\n') line_no_width = len(str(len(line_list))) for line_no, line_txt in enumerate(line_list): @@ -72,6 +77,33 @@ class Suite: result = SuiteResult() runtime = Runtime(module, result.make_imports(), {}) + sys.stderr.write(f'{dashes} Memory (pre run) {dashes}\n') + _dump_memory(runtime.store.mems[0].data) + result.returned_value = runtime.exec('testEntry', args) + sys.stderr.write(f'{dashes} Memory (post run) {dashes}\n') + _dump_memory(runtime.store.mems[0].data) + return result + +def _dump_memory(mem): + line_width = 16 + + prev_line = None + skip = False + for idx in range(0, len(mem), line_width): + line = '' + for idx2 in range(0, line_width): + line += f'{mem[idx + idx2]:02X}' + if idx2 % 2 == 1: + line += ' ' + + if prev_line == line: + if not skip: + sys.stderr.write('**\n') + skip = True + else: + sys.stderr.write(f'{idx:08x} {line}\n') + + prev_line = line diff --git a/tests/integration/test_fib.py b/tests/integration/test_fib.py index bd701c9..b27297d 100644 --- a/tests/integration/test_fib.py +++ b/tests/integration/test_fib.py @@ -28,4 +28,3 @@ def testEntry() -> i32: result = Suite(code_py, 'test_fib').run_code() assert 102334155 == result.returned_value - assert False diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index c399e86..67fd098 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -236,6 +236,7 @@ def helper(left: i32, right: i32) -> i32: assert [] == result.log_int32_list @pytest.mark.integration_test +@pytest.mark.skip('Not yet implemented') def test_assign(): code_py = """ @@ -269,5 +270,5 @@ def helper(shape: Rectangle) -> i32: result = Suite(code_py, 'test_call').run_code() - assert 100 == result.returned_value + assert 252 == result.returned_value assert [] == result.log_int32_list From efe24fb4b542f43a690f978543fe67d2329d957f Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 5 Mar 2022 11:42:01 +0100 Subject: [PATCH 19/73] Locals support in wasm. You can now have multiple objects [skip-ci] --- py2wasm/python.py | 18 ++++++++++++++++-- py2wasm/wasm.py | 9 ++++++--- tests/integration/test_simple.py | 27 ++++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/py2wasm/python.py b/py2wasm/python.py index a0f6e6c..8e5876c 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -35,9 +35,19 @@ class Visitor: [ ('alloc_size', 'i32'), ], + [ + ('result', 'i32'), + ], 'i32', [ - wasm.Statement('i32.const', '4', comment='Stub') + wasm.Statement('i32.const', '0'), + wasm.Statement('i32.const', '0'), + wasm.Statement('i32.load'), + wasm.Statement('local.tee', '$result', comment='Address for this call'), + wasm.Statement('local.get', '$alloc_size'), + wasm.Statement('i32.add'), + wasm.Statement('i32.store', comment='Address for the next call'), + wasm.Statement('local.get', '$result'), ] )) @@ -141,7 +151,11 @@ class Visitor: else: params.append((arg.arg, _parse_annotation(arg.annotation), )) - return wasm.Function(node.name, exported, params, result, []) + locals_ = [ + ('___new_reference___addr', 'i32'), # For the ___new_reference__ method + ] + + return wasm.Function(node.name, exported, params, locals_, result, []) def parse_FunctionDef_body( self, diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py index 338e5a0..b025617 100644 --- a/py2wasm/wasm.py +++ b/py2wasm/wasm.py @@ -61,12 +61,14 @@ class Function: name: str, exported: bool, params: Iterable[Param], + locals_: Iterable[Param], result: Optional[str], statements: Iterable[Statement], ) -> None: self.name = name self.exported = exported self.params = [*params] + self.locals = [*locals_] self.result = result self.statements = [*statements] @@ -77,12 +79,13 @@ class Function: header = ('(export "{}")' if self.exported else '${}').format(self.name) for nam, typ in self.params: - header += ' (param ${} {})'.format(nam, typ) + header += f' (param ${nam} {typ})' if self.result: - header += ' (result {})'.format(self.result) + header += f' (result {self.result})' - header += ' (local $___new_reference___addr i32)' + for nam, typ in self.locals: + header += f' (local ${nam} {typ})' return '(func {}\n {}\n )'.format( header, diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index 67fd098..ca70345 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -252,7 +252,7 @@ def testEntry() -> i32: assert [] == result.log_int32_list @pytest.mark.integration_test -def test_struct(): +def test_struct_1(): code_py = """ class Rectangle: @@ -272,3 +272,28 @@ def helper(shape: Rectangle) -> i32: assert 252 == result.returned_value assert [] == result.log_int32_list + +@pytest.mark.integration_test +def test_struct_2(): + code_py = """ + +class Rectangle: + height: i32 + width: i32 + border: i32 # = 5 + +@exported +def testEntry() -> i32: + return helper(Rectangle(100, 150, 2), Rectangle(200, 90, 3)) + +def helper(shape1: Rectangle, shape2: Rectangle) -> i32: + return ( + shape1.height + shape1.width + shape1.border + + shape2.height + shape2.width + shape2.border + ) +""" + + result = Suite(code_py, 'test_call').run_code() + + assert 545 == result.returned_value + assert [] == result.log_int32_list From c16eb86e1089f43daea1f3ecb5bf5015e4001c99 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Fri, 29 Apr 2022 12:00:38 +0200 Subject: [PATCH 20/73] Adds type class, making it easier to lookup types --- py2wasm/python.py | 231 +++++++++++++++---------------- py2wasm/wasm.py | 130 +++++++++++------ tests/integration/test_simple.py | 20 +++ 3 files changed, 222 insertions(+), 159 deletions(-) diff --git a/py2wasm/python.py b/py2wasm/python.py index 8e5876c..41f7978 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -12,7 +12,7 @@ from . import wasm StatementGenerator = Generator[wasm.Statement, None, None] -WLocals = Dict[str, str] +WLocals = Dict[str, wasm.OurType] class Visitor: """ @@ -21,6 +21,29 @@ class Visitor: Since we need to visit the whole tree, there's no point in subclassing the buildin visitor """ + + def __init__(self) -> None: + self._type_map: WLocals = { + 'None': wasm.OurTypeNone(), + 'bool': wasm.OurTypeBool(), + 'i32': wasm.OurTypeInt32(), + 'i64': wasm.OurTypeInt64(), + 'f32': wasm.OurTypeFloat32(), + 'f64': wasm.OurTypeFloat64(), + } + + def _get_type(self, name: Union[None, str, ast.expr]) -> wasm.OurType: + if name is None: + name = 'None' + + if isinstance(name, ast.expr): + if isinstance(name, ast.Name): + name = name.id + else: + raise NotImplementedError(f'_get_type(ast.{type(name).__name__})') + + return self._type_map[name] + def visit_Module(self, node: ast.Module) -> wasm.Module: """ Visits a Python module, which results in a wasm Module @@ -33,12 +56,12 @@ class Visitor: '___new_reference___', False, [ - ('alloc_size', 'i32'), + ('alloc_size', self._get_type('i32')), ], [ - ('result', 'i32'), + ('result', self._get_type('i32')), ], - 'i32', + self._get_type('i32'), [ wasm.Statement('i32.const', '0'), wasm.Statement('i32.const', '0'), @@ -69,7 +92,7 @@ class Visitor: if isinstance(stmt, ast.ClassDef): wclass = self.pre_visit_ClassDef(module, stmt) - module.classes.append(wclass) + self._type_map[wclass.name] = wclass continue # No other pre visits to do @@ -122,13 +145,13 @@ class Visitor: assert not call.keywords - mod, name, intname = _parse_import_decorator(node.name, call.args) + mod, name, intname = self._parse_import_decorator(node.name, call.args) - return wasm.Import(mod, name, intname, _parse_import_args(node.args)) + return wasm.Import(mod, name, intname, self._parse_import_args(node.args)) else: raise NotImplementedError - result = None if node.returns is None else _parse_annotation(node.returns) + result = None if node.returns is None else self._get_type(node.returns) assert not node.args.vararg assert not node.args.kwonlyargs @@ -136,23 +159,12 @@ class Visitor: assert not node.args.kwarg assert not node.args.defaults - class_lookup = { - x.name: x - for x in module.classes - } - - params = [] + params: List[wasm.Param] = [] for arg in [*node.args.posonlyargs, *node.args.args]: - if not isinstance(arg.annotation, ast.Name): - raise NotImplementedError + params.append((arg.arg, self._get_type(arg.annotation), )) - if arg.annotation.id in class_lookup: - params.append((arg.arg, 'i32', )) - else: - params.append((arg.arg, _parse_annotation(arg.annotation), )) - - locals_ = [ - ('___new_reference___addr', 'i32'), # For the ___new_reference__ method + locals_: List[wasm.Param] = [ + ('___new_reference___addr', self._get_type('i32')), # For the ___new_reference__ method ] return wasm.Function(node.name, exported, params, locals_, result, []) @@ -180,7 +192,7 @@ class Visitor: self, module: wasm.Module, node: ast.ClassDef, - ) -> wasm.Class: + ) -> wasm.OurTypeClass: """ TODO: Document this """ @@ -200,7 +212,7 @@ class Visitor: raise NotImplementedError if not isinstance(stmt.annotation, ast.Name): - raise NotImplementedError + raise NotImplementedError('Cannot recurse classes yet') if stmt.annotation.id != 'i32': raise NotImplementedError @@ -217,13 +229,13 @@ class Visitor: default = wasm.Constant(stmt.value.value) member = wasm.ClassMember( - stmt.target.id, stmt.annotation.id, offset, default + stmt.target.id, self._get_type(stmt.annotation), offset, default ) members.append(member) - offset += member.alloc_size() + offset += member.type.alloc_size() - return wasm.Class(node.name, members) + return wasm.OurTypeClass(node.name, members) def visit_stmt( self, @@ -241,7 +253,7 @@ class Visitor: if isinstance(stmt, ast.Expr): assert isinstance(stmt.value, ast.Call) - return self.visit_Call(module, wlocals, "None", stmt.value) + return self.visit_Call(module, wlocals, self._get_type('None'), stmt.value) if isinstance(stmt, ast.If): return self.visit_If(module, func, wlocals, stmt) @@ -261,7 +273,7 @@ class Visitor: assert stmt.value is not None - return_type = _parse_annotation(func.returns) + return_type = self._get_type(func.returns) yield from self.visit_expr(module, wlocals, return_type, stmt.value) yield wasm.Statement('return') @@ -279,7 +291,7 @@ class Visitor: yield from self.visit_expr( module, wlocals, - 'bool', + self._get_type('bool'), stmt.test, ) @@ -299,7 +311,7 @@ class Visitor: self, module: wasm.Module, wlocals: WLocals, - exp_type: str, + exp_type: wasm.OurType, node: ast.expr, ) -> StatementGenerator: """ @@ -332,7 +344,7 @@ class Visitor: self, module: wasm.Module, wlocals: WLocals, - exp_type: str, + exp_type: wasm.OurType, node: ast.UnaryOp, ) -> StatementGenerator: """ @@ -356,7 +368,7 @@ class Visitor: self, module: wasm.Module, wlocals: WLocals, - exp_type: str, + exp_type: wasm.OurType, node: ast.BinOp, ) -> StatementGenerator: """ @@ -366,11 +378,11 @@ class Visitor: yield from self.visit_expr(module, wlocals, exp_type, node.right) if isinstance(node.op, ast.Add): - yield wasm.Statement('{}.add'.format(exp_type)) + yield wasm.Statement('{}.add'.format(exp_type.to_wasm())) return if isinstance(node.op, ast.Sub): - yield wasm.Statement('{}.sub'.format(exp_type)) + yield wasm.Statement('{}.sub'.format(exp_type.to_wasm())) return raise NotImplementedError(node.op) @@ -379,19 +391,19 @@ class Visitor: self, module: wasm.Module, wlocals: WLocals, - exp_type: str, + exp_type: wasm.OurType, node: ast.Compare, ) -> StatementGenerator: """ Visits a Compare node as (part of) an expression """ - assert 'bool' == exp_type + assert isinstance(exp_type, wasm.OurTypeBool) if 1 != len(node.ops) or 1 != len(node.comparators): raise NotImplementedError - yield from self.visit_expr(module, wlocals, 'i32', node.left) - yield from self.visit_expr(module, wlocals, 'i32', node.comparators[0]) + yield from self.visit_expr(module, wlocals, self._get_type('i32'), node.left) + yield from self.visit_expr(module, wlocals, self._get_type('i32'), node.comparators[0]) if isinstance(node.ops[0], ast.Lt): yield wasm.Statement('i32.lt_s') @@ -411,7 +423,7 @@ class Visitor: self, module: wasm.Module, wlocals: WLocals, - exp_type: str, + exp_type: wasm.OurType, node: ast.Call, ) -> StatementGenerator: """ @@ -422,11 +434,15 @@ class Visitor: called_name = node.func.id - search_list: List[Union[wasm.Function, wasm.Import, wasm.Class]] + if called_name in self._type_map: + klass = self._type_map[called_name] + if isinstance(klass, wasm.OurTypeClass): + return self.visit_Call_class(module, wlocals, exp_type, node, klass) + + search_list: List[Union[wasm.Function, wasm.Import]] search_list = [ *module.functions, *module.imports, - *module.classes, ] called_func_list = [ @@ -438,18 +454,15 @@ class Visitor: assert 1 == len(called_func_list), \ 'Could not find function {}'.format(node.func.id) - if isinstance(called_func_list[0], wasm.Class): - return self.visit_Call_class(module, wlocals, exp_type, node, called_func_list[0]) - return self.visit_Call_func(module, wlocals, exp_type, node, called_func_list[0]) def visit_Call_class( self, module: wasm.Module, wlocals: WLocals, - exp_type: str, + exp_type: wasm.OurType, node: ast.Call, - cls: wasm.Class, + cls: wasm.OurTypeClass, ) -> StatementGenerator: """ Visits a Call node as (part of) an expression @@ -469,8 +482,8 @@ class Visitor: raise NotImplementedError yield wasm.Statement('local.get', '$___new_reference___addr') - yield wasm.Statement(f'{member.type}.const', str(arg.value)) - yield wasm.Statement(f'{member.type}.store', 'offset=' + str(member.offset)) + yield wasm.Statement(f'{member.type.to_wasm()}.const', str(arg.value)) + yield wasm.Statement(f'{member.type.to_wasm()}.store', 'offset=' + str(member.offset)) yield wasm.Statement('local.get', '$___new_reference___addr') @@ -478,7 +491,7 @@ class Visitor: self, module: wasm.Module, wlocals: WLocals, - exp_type: str, + exp_type: wasm.OurType, node: ast.Call, func: Union[wasm.Function, wasm.Import], ) -> StatementGenerator: @@ -491,7 +504,7 @@ class Visitor: called_params = func.params called_result = func.result - assert exp_type == called_result + assert exp_type == called_result, 'Function does not match expected type' assert len(called_params) == len(node.args), \ '{}:{} Function {} requires {} arguments, but {} are supplied'.format( @@ -507,32 +520,32 @@ class Visitor: '${}'.format(called_name), ) - def visit_Constant(self, exp_type: str, node: ast.Constant) -> StatementGenerator: + def visit_Constant(self, exp_type: wasm.OurType, node: ast.Constant) -> StatementGenerator: """ Visits a Constant node as (part of) an expression """ - if 'i32' == exp_type: + if isinstance(exp_type, wasm.OurTypeInt32): assert isinstance(node.value, int) assert -2147483648 <= node.value <= 2147483647 yield wasm.Statement('i32.const', str(node.value)) return - if 'i64' == exp_type: + if isinstance(exp_type, wasm.OurTypeInt64): assert isinstance(node.value, int) assert -9223372036854775808 <= node.value <= 9223372036854775807 yield wasm.Statement('i64.const', str(node.value)) return - if 'f32' == exp_type: + if isinstance(exp_type, wasm.OurTypeFloat32): assert isinstance(node.value, float) # TODO: Size check? yield wasm.Statement('f32.const', node.value.hex()) return - if 'f64' == exp_type: + if isinstance(exp_type, wasm.OurTypeFloat64): assert isinstance(node.value, float) # TODO: Size check? @@ -545,27 +558,21 @@ class Visitor: self, module: wasm.Module, wlocals: WLocals, - exp_type: str, + exp_type: wasm.OurType, node: ast.Attribute, ) -> StatementGenerator: """ Visits an Attribute node as (part of) an expression """ - if not isinstance(node.value, ast.Name): raise NotImplementedError if not isinstance(node.ctx, ast.Load): raise NotImplementedError - cls_list = [ - x - for x in module.classes - if x.name == 'Rectangle' # TODO: STUB, since we can't acces the type properly - ] + cls = wlocals[node.value.id] - assert len(cls_list) == 1 - cls = cls_list[0] + assert isinstance(cls, wasm.OurTypeClass), f'Cannot take property of {cls}' member_list = [ x @@ -577,79 +584,71 @@ class Visitor: member = member_list[0] yield wasm.Statement('local.get', '$' + node.value.id) - yield wasm.Statement(exp_type + '.load', 'offset=' + str(member.offset)) + yield wasm.Statement(exp_type.to_wasm() + '.load', 'offset=' + str(member.offset)) - def visit_Name(self, wlocals: WLocals, exp_type: str, node: ast.Name) -> StatementGenerator: + def visit_Name(self, wlocals: WLocals, exp_type: wasm.OurType, node: ast.Name) -> StatementGenerator: """ Visits a Name node as (part of) an expression """ assert node.id in wlocals - if exp_type == 'i64' and wlocals[node.id] == 'i32': + if (isinstance(exp_type, wasm.OurTypeInt64) + and isinstance(wlocals[node.id], wasm.OurTypeInt32)): yield wasm.Statement('local.get', '${}'.format(node.id)) yield wasm.Statement('i64.extend_i32_s') return - if exp_type == 'f64' and wlocals[node.id] == 'f32': + if (isinstance(exp_type, wasm.OurTypeFloat64) + and isinstance(wlocals[node.id], wasm.OurTypeFloat32)): yield wasm.Statement('local.get', '${}'.format(node.id)) yield wasm.Statement('f64.promote_f32') return - assert exp_type == wlocals[node.id] + assert exp_type == wlocals[node.id], (exp_type, wlocals[node.id], ) yield wasm.Statement( 'local.get', '${}'.format(node.id), ) -def _parse_import_decorator(func_name: str, args: List[ast.expr]) -> Tuple[str, str, str]: - """ - Parses an @import decorator - """ + def _parse_import_decorator(self, func_name: str, args: List[ast.expr]) -> Tuple[str, str, str]: + """ + Parses an @import decorator + """ - assert 0 < len(args) < 3 - str_args = [ - arg.value - for arg in args - if isinstance(arg, ast.Constant) - and isinstance(arg.value, str) - ] - assert len(str_args) == len(args) + assert 0 < len(args) < 3 + str_args = [ + arg.value + for arg in args + if isinstance(arg, ast.Constant) + and isinstance(arg.value, str) + ] + assert len(str_args) == len(args) - module = str_args.pop(0) - if str_args: - name = str_args.pop(0) - else: - name = func_name + module = str_args.pop(0) + if str_args: + name = str_args.pop(0) + else: + name = func_name - return module, name, func_name + return module, name, func_name -def _parse_import_args(args: ast.arguments) -> List[Tuple[str, str]]: - """ - Parses the arguments for an @imported method - """ + def _parse_import_args(self, args: ast.arguments) -> List[wasm.Param]: + """ + Parses the arguments for an @imported method + """ - assert not args.vararg - assert not args.kwonlyargs - assert not args.kw_defaults - assert not args.kwarg - assert not args.defaults # Maybe support this later on + assert not args.vararg + assert not args.kwonlyargs + assert not args.kw_defaults + assert not args.kwarg + assert not args.defaults # Maybe support this later on - arg_list = [ - *args.posonlyargs, - *args.args, - ] + arg_list = [ + *args.posonlyargs, + *args.args, + ] - return [ - (arg.arg, _parse_annotation(arg.annotation)) - for arg in arg_list - ] - -def _parse_annotation(ann: Optional[ast.expr]) -> str: - """ - Parses a typing annotation - """ - assert ann is not None, 'Web Assembly requires type annotations' - assert isinstance(ann, ast.Name) - result = ann.id - assert result in ['i32', 'i64', 'f32', 'f64'], result - return result + return [ + (arg.arg, self._get_type(arg.annotation)) + for arg in arg_list + ] diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py index b025617..1939301 100644 --- a/py2wasm/wasm.py +++ b/py2wasm/wasm.py @@ -2,9 +2,90 @@ Python classes for storing the representation of Web Assembly code """ -from typing import Iterable, List, Optional, Tuple, Union +from typing import Any, Iterable, List, Optional, Tuple, Union -Param = Tuple[str, str] +### +### This part is more intermediate code +### + +class OurType: + def to_wasm(self) -> str: + raise NotImplementedError + + def alloc_size(self) -> int: + raise NotImplementedError + +class OurTypeNone(OurType): + pass + +class OurTypeBool(OurType): + pass + +class OurTypeInt32(OurType): + def to_wasm(self) -> str: + return 'i32' + + def alloc_size(self) -> int: + return 4 + +class OurTypeInt64(OurType): + def to_wasm(self) -> str: + return 'i64' + + def alloc_size(self) -> int: + return 8 + +class OurTypeFloat32(OurType): + def to_wasm(self) -> str: + return 'f32' + + def alloc_size(self) -> int: + return 4 + +class OurTypeFloat64(OurType): + def to_wasm(self) -> str: + return 'f64' + + def alloc_size(self) -> int: + return 8 + +class Constant: + """ + TODO + """ + def __init__(self, value: Union[None, bool, int, float]) -> None: + self.value = value + +class ClassMember: + """ + Represents a class member + """ + def __init__(self, name: str, type_: OurType, offset: int, default: Optional[Constant]) -> None: + self.name = name + self.type = type_ + self.offset = offset + self.default = default + +class OurTypeClass(OurType): + """ + Represents a class + """ + def __init__(self, name: str, members: List[ClassMember]) -> None: + self.name = name + self.members = members + + def to_wasm(self) -> str: + return 'i32' # WASM uses 32 bit pointers + + def alloc_size(self) -> int: + return sum(x.type.alloc_size() for x in self.members) + + +Param = Tuple[str, OurType] + +### +## This part is more actual web assembly +### class Import: """ @@ -62,7 +143,7 @@ class Function: exported: bool, params: Iterable[Param], locals_: Iterable[Param], - result: Optional[str], + result: Optional[OurType], statements: Iterable[Statement], ) -> None: self.name = name @@ -79,55 +160,19 @@ class Function: header = ('(export "{}")' if self.exported else '${}').format(self.name) for nam, typ in self.params: - header += f' (param ${nam} {typ})' + header += f' (param ${nam} {typ.to_wasm()})' if self.result: - header += f' (result {self.result})' + header += f' (result {self.result.to_wasm()})' for nam, typ in self.locals: - header += f' (local ${nam} {typ})' + header += f' (local ${nam} {typ.to_wasm()})' return '(func {}\n {}\n )'.format( header, '\n '.join(x.generate() for x in self.statements), ) -class Constant: - """ - TODO - """ - def __init__(self, value: Union[None, bool, int, float]) -> None: - self.value = value - -class ClassMember: - """ - Represents a Web Assembly class member - """ - def __init__(self, name: str, type_: str, offset: int, default: Optional[Constant]) -> None: - self.name = name - self.type = type_ - self.offset = offset - self.default = default - - def alloc_size(self) -> int: - SIZE_MAP = { - 'i32': 4, - 'i64': 4, - } - - return SIZE_MAP[self.type] - -class Class: - """ - Represents a Web Assembly class - """ - def __init__(self, name: str, members: List[ClassMember]) -> None: - self.name = name - self.members = members - - def alloc_size(self) -> int: - return sum(x.alloc_size() for x in self.members) - class Module: """ Represents a Web Assembly module @@ -135,7 +180,6 @@ class Module: def __init__(self) -> None: self.imports: List[Import] = [] self.functions: List[Function] = [] - self.classes: List[Class] = [] def generate(self) -> str: """ diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index ca70345..dca3291 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -251,6 +251,26 @@ def testEntry() -> i32: assert 8947 == result.returned_value assert [] == result.log_int32_list +@pytest.mark.integration_test +def test_struct_0(): + code_py = """ + +class CheckedValue: + value: i32 + +@exported +def testEntry() -> i32: + return helper(CheckedValue(2345)) + +def helper(cv: CheckedValue) -> i32: + return cv.value +""" + + result = Suite(code_py, 'test_call').run_code() + + assert 2345 == result.returned_value + assert [] == result.log_int32_list + @pytest.mark.integration_test def test_struct_1(): code_py = """ From c5d039aa1f7d981365b5aac5c0b36e847574ee01 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Fri, 29 Apr 2022 12:30:31 +0200 Subject: [PATCH 21/73] Implements tuple creation / usage --- py2wasm/python.py | 100 ++++++++++++++++++++++++++++++- py2wasm/wasm.py | 21 +++++++ tests/integration/test_simple.py | 17 ++++++ 3 files changed, 135 insertions(+), 3 deletions(-) diff --git a/py2wasm/python.py b/py2wasm/python.py index 41f7978..6d52490 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -39,6 +39,23 @@ class Visitor: if isinstance(name, ast.expr): if isinstance(name, ast.Name): name = name.id + elif isinstance(name, ast.Subscript) and isinstance(name.value, ast.Name) and name.value.id == 'Tuple': + assert isinstance(name.ctx, ast.Load) + assert isinstance(name.slice, ast.Index) + assert isinstance(name.slice.value, ast.Tuple) + + args: List[wasm.TupleMember] = [] + offset = 0 + for name_arg in name.slice.value.elts: + arg = wasm.TupleMember( + self._get_type(name_arg), + offset + ) + + args.append(arg) + offset += arg.type.alloc_size() + + return wasm.OurTypeTuple(args) else: raise NotImplementedError(f'_get_type(ast.{type(name).__name__})') @@ -332,11 +349,17 @@ class Visitor: if isinstance(node, ast.Constant): return self.visit_Constant(exp_type, node) + if isinstance(node, ast.Attribute): + return self.visit_Attribute(module, wlocals, exp_type, node) + + if isinstance(node, ast.Subscript): + return self.visit_Subscript(module, wlocals, exp_type, node) + if isinstance(node, ast.Name): return self.visit_Name(wlocals, exp_type, node) - if isinstance(node, ast.Attribute): - return self.visit_Attribute(module, wlocals, exp_type, node) + if isinstance(node, ast.Tuple): + return self.visit_Tuple(wlocals, exp_type, node) raise NotImplementedError(node) @@ -470,7 +493,7 @@ class Visitor: This instantiates the class """ - # TODO: malloc call + # TODO: free call yield wasm.Statement('i32.const', str(cls.alloc_size())) yield wasm.Statement('call', '$___new_reference___') @@ -586,6 +609,77 @@ class Visitor: yield wasm.Statement('local.get', '$' + node.value.id) yield wasm.Statement(exp_type.to_wasm() + '.load', 'offset=' + str(member.offset)) + def visit_Subscript( + self, + module: wasm.Module, + wlocals: WLocals, + exp_type: wasm.OurType, + node: ast.Subscript, + ) -> StatementGenerator: + """ + Visits an Subscript node as (part of) an expression + """ + if not isinstance(node.value, ast.Name): + raise NotImplementedError + + if not isinstance(node.slice, ast.Index): + raise NotImplementedError + + if not isinstance(node.slice.value, ast.Constant): + raise NotImplementedError + + if not isinstance(node.slice.value.value, int): + raise NotImplementedError + + if not isinstance(node.ctx, ast.Load): + raise NotImplementedError + + tpl_idx = node.slice.value.value + + tpl = wlocals[node.value.id] + + assert isinstance(tpl, wasm.OurTypeTuple), f'Cannot take index of {tpl}' + + assert 0 <= tpl_idx < len(tpl.members), \ + f'Tuple index out of bounds; {node.value.id} has {len(tpl.members)} members' + + member = tpl.members[tpl_idx] + + yield wasm.Statement('local.get', '$' + node.value.id) + yield wasm.Statement(exp_type.to_wasm() + '.load', 'offset=' + str(member.offset)) + + def visit_Tuple( + self, + wlocals: WLocals, + exp_type: wasm.OurType, + node: ast.Tuple, + ) -> StatementGenerator: + """ + Visits an Tuple node as (part of) an expression + """ + assert isinstance(exp_type, wasm.OurTypeTuple), 'Expression is not expecting a tuple' + + # TODO: free call + + tpl = exp_type + + yield wasm.Statement('nop', comment='Start tuple allocation') + + yield wasm.Statement('i32.const', str(tpl.alloc_size())) + yield wasm.Statement('call', '$___new_reference___') + yield wasm.Statement('local.set', '$___new_reference___addr', comment='Allocate for tuple') + + for member, arg in zip(tpl.members, node.elts): + if not isinstance(arg, ast.Constant): + raise NotImplementedError('TODO: Non-const tuple members') + + yield wasm.Statement('local.get', '$___new_reference___addr') + yield wasm.Statement(f'{member.type.to_wasm()}.const', str(arg.value)) + yield wasm.Statement(f'{member.type.to_wasm()}.store', 'offset=' + str(member.offset), + comment='Write tuple value to memory') + + yield wasm.Statement('local.get', '$___new_reference___addr', comment='Store tuple address on stack') + def visit_Name(self, wlocals: WLocals, exp_type: wasm.OurType, node: ast.Name) -> StatementGenerator: """ Visits a Name node as (part of) an expression diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py index 1939301..2aa4ac8 100644 --- a/py2wasm/wasm.py +++ b/py2wasm/wasm.py @@ -80,6 +80,27 @@ class OurTypeClass(OurType): def alloc_size(self) -> int: return sum(x.type.alloc_size() for x in self.members) +class TupleMember: + """ + Represents a tuple member + """ + def __init__(self, type_: OurType, offset: int) -> None: + self.type = type_ + self.offset = offset + +class OurTypeTuple(OurType): + """ + Represents a tuple + """ + def __init__(self, members: List[TupleMember]) -> None: + self.members = members + + def to_wasm(self) -> str: + return 'i32' # WASM uses 32 bit pointers + + def alloc_size(self) -> int: + return sum(x.type.alloc_size() for x in self.members) + Param = Tuple[str, OurType] diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index dca3291..0772cca 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -317,3 +317,20 @@ def helper(shape1: Rectangle, shape2: Rectangle) -> i32: assert 545 == result.returned_value assert [] == result.log_int32_list + +@pytest.mark.integration_test +def test_tuple_int(): + code_py = """ + +@exported +def testEntry() -> i32: + return helper((24, 57, 80, )) + +def helper(vector: Tuple[i32, i32, i32]) -> i32: + return vector[0] + vector[1] + vector[2] +""" + + result = Suite(code_py, 'test_call').run_code() + + assert 161 == result.returned_value + assert [] == result.log_int32_list From 249c00f6a2609182a82c7c42f46665298ad6103c Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Fri, 29 Apr 2022 12:56:45 +0200 Subject: [PATCH 22/73] Implements mult and sqrt --- py2wasm/python.py | 22 ++++++++++++++++++++++ tests/integration/test_simple.py | 19 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/py2wasm/python.py b/py2wasm/python.py index 6d52490..7639ee1 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -91,6 +91,24 @@ class Visitor: ] )) + # We create functions for these special instructions + # We'll let a future inline optimizer clean this up for us + # TODO: Either a) make a sqrt32 and a sqrt64; + # or b) decide to make the whole language auto-cast + module.functions.append(wasm.Function( + 'sqrt', + False, + [ + ('z', self._type_map['f32']), + ], + [], + self._type_map['f32'], + [ + wasm.Statement('local.get', '$z'), + wasm.Statement('f32.sqrt'), + ] + )) + # Do a check first for all function definitions # to get their types. Otherwise you cannot call # a method that you haven't defined just yet, @@ -404,6 +422,10 @@ class Visitor: yield wasm.Statement('{}.add'.format(exp_type.to_wasm())) return + if isinstance(node.op, ast.Mult): + yield wasm.Statement('{}.mul'.format(exp_type.to_wasm())) + return + if isinstance(node.op, ast.Sub): yield wasm.Statement('{}.sub'.format(exp_type.to_wasm())) return diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index 0772cca..fd89412 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -334,3 +334,22 @@ def helper(vector: Tuple[i32, i32, i32]) -> i32: assert 161 == result.returned_value assert [] == result.log_int32_list + +@pytest.mark.integration_test +def test_tuple_float(): + code_py = """ + +@exported +def testEntry() -> f32: + return helper((1, 2, 3, )) + +def helper(v: Tuple[f32, f32, f32]) -> f32: + # sqrt is guaranteed by wasm, so we can use it + # ATM it's a 32 bit variant, but that might change. + return sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]) +""" + + result = Suite(code_py, 'test_call').run_code() + + assert 3.74 < result.returned_value < 3.75 + assert [] == result.log_int32_list From 6b717dba0be3fd230f3b26a666d59fdfe2e275cb Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 7 May 2022 12:14:34 +0200 Subject: [PATCH 23/73] Fix pathing issue when wat2wasm is not globally installed --- Makefile | 2 +- tests/integration/helpers.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index eef7502..36d0f06 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ server: python3.8 -m http.server test: venv/.done - venv/bin/pytest tests $(TEST_FLAGS) + WAT2WASM=$(WAT2WASM) venv/bin/pytest tests $(TEST_FLAGS) lint: venv/.done venv/bin/pylint py2wasm diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 444e1c0..e434d05 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -1,6 +1,8 @@ import io +import os import subprocess import sys + from tempfile import NamedTemporaryFile from pywasm import binary @@ -9,14 +11,16 @@ from pywasm import Runtime from py2wasm.utils import process def wat2wasm(code_wat): + path = os.environ.get('WAT2WASM', 'wat2wasm') + with NamedTemporaryFile('w+t') as input_fp: input_fp.write(code_wat) input_fp.flush() with NamedTemporaryFile('w+b') as output_fp: - result = subprocess.run( + subprocess.run( [ - 'wat2wasm', + path, input_fp.name, '-o', output_fp.name, From 865eccd719bd99164ef0e03ff08a29034a51b3f1 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 7 May 2022 14:21:43 +0200 Subject: [PATCH 24/73] Testing with various wasm implementations Also: - Started on SIMD before finding out no implementation supports that yet - Type fix result Import / Function - Various error reporting improvements - Non-const tuple members --- Makefile | 1 + py2wasm/python.py | 36 ++++++++--- py2wasm/wasm.py | 45 +++++++++++-- requirements.txt | 4 ++ tests/integration/helpers.py | 104 ++++++++++++++++++++++++++----- tests/integration/test_simple.py | 15 ++++- 6 files changed, 177 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index 36d0f06..d952e0f 100644 --- a/Makefile +++ b/Makefile @@ -29,5 +29,6 @@ typecheck: venv/.done venv/.done: requirements.txt python3.8 -m venv venv + venv/bin/python3 -m pip install wheel venv/bin/python3 -m pip install -r $^ touch $@ diff --git a/py2wasm/python.py b/py2wasm/python.py index 7639ee1..4867061 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -30,6 +30,8 @@ class Visitor: 'i64': wasm.OurTypeInt64(), 'f32': wasm.OurTypeFloat32(), 'f64': wasm.OurTypeFloat64(), + + 'i32x4': wasm.OurTypeVectorInt32x4(), } def _get_type(self, name: Union[None, str, ast.expr]) -> wasm.OurType: @@ -186,7 +188,7 @@ class Visitor: else: raise NotImplementedError - result = None if node.returns is None else self._get_type(node.returns) + result = self._get_type(node.returns) assert not node.args.vararg assert not node.args.kwonlyargs @@ -377,7 +379,7 @@ class Visitor: return self.visit_Name(wlocals, exp_type, node) if isinstance(node, ast.Tuple): - return self.visit_Tuple(wlocals, exp_type, node) + return self.visit_Tuple(module, wlocals, exp_type, node) raise NotImplementedError(node) @@ -549,7 +551,8 @@ class Visitor: called_params = func.params called_result = func.result - assert exp_type == called_result, 'Function does not match expected type' + assert exp_type == called_result, \ + f'Function does not match expected type; {called_name} returns {called_result.to_python()} but {exp_type.to_python()} is expected' assert len(called_params) == len(node.args), \ '{}:{} Function {} requires {} arguments, but {} are supplied'.format( @@ -584,7 +587,8 @@ class Visitor: return if isinstance(exp_type, wasm.OurTypeFloat32): - assert isinstance(node.value, float) + assert isinstance(node.value, float), \ + f'Expression expected a float, but received {node.value}' # TODO: Size check? yield wasm.Statement('f32.const', node.value.hex()) @@ -672,6 +676,7 @@ class Visitor: def visit_Tuple( self, + module: wasm.Module, wlocals: WLocals, exp_type: wasm.OurType, node: ast.Tuple, @@ -679,7 +684,21 @@ class Visitor: """ Visits an Tuple node as (part of) an expression """ - assert isinstance(exp_type, wasm.OurTypeTuple), 'Expression is not expecting a tuple' + if isinstance(exp_type, wasm.OurTypeVector): + assert isinstance(exp_type, wasm.OurTypeVectorInt32x4), 'Not implemented yet' + assert len(node.elts) == 4, 'Not implemented yet' + + values: List[str] = [] + for arg in node.elts: + assert isinstance(arg, ast.Constant) + assert isinstance(arg.value, int) + values.append(str(arg.value)) + + yield wasm.Statement('v128.const', 'i32x4', *values) + return + + assert isinstance(exp_type, wasm.OurTypeTuple), \ + f'Expression is expecting {exp_type}, not a tuple' # TODO: free call @@ -692,11 +711,10 @@ class Visitor: yield wasm.Statement('local.set', '$___new_reference___addr', comment='Allocate for tuple') for member, arg in zip(tpl.members, node.elts): - if not isinstance(arg, ast.Constant): - raise NotImplementedError('TODO: Non-const tuple members') - yield wasm.Statement('local.get', '$___new_reference___addr') - yield wasm.Statement(f'{member.type.to_wasm()}.const', str(arg.value)) + + yield from self.visit_expr(module, wlocals, member.type, arg) + yield wasm.Statement(f'{member.type.to_wasm()}.store', 'offset=' + str(member.offset), comment='Write tuple value to memory') diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py index 2aa4ac8..b9b7d85 100644 --- a/py2wasm/wasm.py +++ b/py2wasm/wasm.py @@ -9,11 +9,14 @@ from typing import Any, Iterable, List, Optional, Tuple, Union ### class OurType: + def to_python(self) -> str: + raise NotImplementedError(self, 'to_python') + def to_wasm(self) -> str: - raise NotImplementedError + raise NotImplementedError(self, 'to_wasm') def alloc_size(self) -> int: - raise NotImplementedError + raise NotImplementedError(self, 'alloc_size') class OurTypeNone(OurType): pass @@ -22,6 +25,9 @@ class OurTypeBool(OurType): pass class OurTypeInt32(OurType): + def to_python(self) -> str: + return 'i32' + def to_wasm(self) -> str: return 'i32' @@ -49,6 +55,21 @@ class OurTypeFloat64(OurType): def alloc_size(self) -> int: return 8 +class OurTypeVector(OurType): + """ + A vector is a 128-bit value + """ + def to_wasm(self) -> str: + return 'v128' + + def alloc_size(self) -> int: + return 16 + +class OurTypeVectorInt32x4(OurTypeVector): + """ + 4 Int32 values in a single vector + """ + class Constant: """ TODO @@ -95,12 +116,28 @@ class OurTypeTuple(OurType): def __init__(self, members: List[TupleMember]) -> None: self.members = members + def to_python(self) -> str: + return 'Tuple[' + ( + ', '.join(x.type.to_python() for x in self.members) + ) + ']' + def to_wasm(self) -> str: return 'i32' # WASM uses 32 bit pointers def alloc_size(self) -> int: return sum(x.type.alloc_size() for x in self.members) + def __eq__(self, other: Any) -> bool: + if not isinstance(other, OurTypeTuple): + raise NotImplementedError + + return ( + len(self.members) == len(other.members) + and all( + self.members[x].type == other.members[x].type + for x in range(len(self.members)) + ) + ) Param = Tuple[str, OurType] @@ -123,7 +160,7 @@ class Import: self.name = name self.intname = intname self.params = [*params] - self.result: str = 'None' + self.result: OurType = OurTypeNone() def generate(self) -> str: """ @@ -164,7 +201,7 @@ class Function: exported: bool, params: Iterable[Param], locals_: Iterable[Param], - result: Optional[OurType], + result: OurType, statements: Iterable[Statement], ) -> None: self.name = name diff --git a/requirements.txt b/requirements.txt index c20edb1..a9abf93 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,7 @@ pylint==2.7.4 pytest==6.2.2 pytest-integration==0.2.2 pywasm==1.0.7 +pywasm3==0.5.0 +wasmer==1.0.0 +wasmer_compiler_cranelift==1.0.0 +wasmtime==0.36.0 diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index e434d05..98f5dad 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -5,11 +5,19 @@ import sys from tempfile import NamedTemporaryFile -from pywasm import binary -from pywasm import Runtime +import pywasm + +import wasm3 + +import wasmer +import wasmer_compiler_cranelift + +import wasmtime from py2wasm.utils import process +DASHES = '-' * 16 + def wat2wasm(code_wat): path = os.environ.get('WAT2WASM', 'wat2wasm') @@ -63,9 +71,7 @@ class Suite: """ code_wat = process(self.code_py, self.test_name) - dashes = '-' * 16 - - sys.stderr.write(f'{dashes} Assembly {dashes}\n') + sys.stderr.write(f'{DASHES} Assembly {DASHES}\n') line_list = code_wat.split('\n') line_no_width = len(str(len(line_list))) @@ -76,20 +82,90 @@ class Suite: )) code_wasm = wat2wasm(code_wat) - module = binary.Module.from_reader(io.BytesIO(code_wasm)) - result = SuiteResult() - runtime = Runtime(module, result.make_imports(), {}) + return _run_pywasm3(code_wasm, args) - sys.stderr.write(f'{dashes} Memory (pre run) {dashes}\n') - _dump_memory(runtime.store.mems[0].data) +def _run_pywasm(code_wasm, args): + # https://pypi.org/project/pywasm/ + result = SuiteResult() - result.returned_value = runtime.exec('testEntry', args) + module = pywasm.binary.Module.from_reader(io.BytesIO(code_wasm)) - sys.stderr.write(f'{dashes} Memory (post run) {dashes}\n') - _dump_memory(runtime.store.mems[0].data) + runtime = pywasm.Runtime(module, result.make_imports(), {}) - return result + sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') + _dump_memory(runtime.store.mems[0].data) + + result.returned_value = runtime.exec('testEntry', args) + + sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n') + _dump_memory(runtime.store.mems[0].data) + + return result + +def _run_pywasm3(code_wasm, args): + # https://pypi.org/project/pywasm3/ + result = SuiteResult() + + env = wasm3.Environment() + + mod = env.parse_module(code_wasm) + + rtime = env.new_runtime(1024 * 1024) + rtime.load(mod) + + sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') + _dump_memory(rtime.get_memory(0).tobytes()) + + result.returned_value = rtime.find_function('testEntry')(*args) + + sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n') + _dump_memory(rtime.get_memory(0).tobytes()) + + return result + +def _run_wasmtime(code_wasm, args): + # https://pypi.org/project/wasmtime/ + result = SuiteResult() + + store = wasmtime.Store() + + module = wasmtime.Module(store.engine, code_wasm) + + instance = wasmtime.Instance(store, module, []) + + sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') + sys.stderr.write('\n') + + result.returned_value = instance.exports(store)['testEntry'](store, *args) + + sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n') + sys.stderr.write('\n') + + return result + +def _run_wasmer(code_wasm, args): + # https://pypi.org/project/wasmer/ + result = SuiteResult() + + store = wasmer.Store(wasmer.engine.JIT(wasmer_compiler_cranelift.Compiler)) + + # Let's compile the module to be able to execute it! + module = wasmer.Module(store, code_wasm) + + # Now the module is compiled, we can instantiate it. + instance = wasmer.Instance(module) + + sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') + sys.stderr.write('\n') + + # Call the exported `sum` function. + result.returned_value = instance.exports.testEntry(*args) + + sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n') + sys.stderr.write('\n') + + return result def _dump_memory(mem): line_width = 16 diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index fd89412..0703e08 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -341,7 +341,7 @@ def test_tuple_float(): @exported def testEntry() -> f32: - return helper((1, 2, 3, )) + return helper((1.0, 2.0, 3.0, )) def helper(v: Tuple[f32, f32, f32]) -> f32: # sqrt is guaranteed by wasm, so we can use it @@ -353,3 +353,16 @@ def helper(v: Tuple[f32, f32, f32]) -> f32: assert 3.74 < result.returned_value < 3.75 assert [] == result.log_int32_list + +@pytest.mark.integration_test +@pytest.mark.skip('SIMD support is but a dream') +def test_tuple_i32x4(): + code_py = """ +@exported +def testEntry() -> i32x4: + return (51, 153, 204, 0, ) +""" + + result = Suite(code_py, 'test_rgb2hsl').run_code() + + assert (1, 2, 3, 0) == result.returned_value From efba4e0daa208ae6ff30fb857e74c3fcad4578c7 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Tue, 24 May 2022 11:43:42 +0200 Subject: [PATCH 25/73] pip fix, wasmer update --- Makefile | 2 +- requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d952e0f..5749492 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,6 @@ typecheck: venv/.done venv/.done: requirements.txt python3.8 -m venv venv - venv/bin/python3 -m pip install wheel + venv/bin/python3 -m pip install wheel pip --upgrade venv/bin/python3 -m pip install -r $^ touch $@ diff --git a/requirements.txt b/requirements.txt index a9abf93..b969a61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,6 @@ pytest==6.2.2 pytest-integration==0.2.2 pywasm==1.0.7 pywasm3==0.5.0 -wasmer==1.0.0 -wasmer_compiler_cranelift==1.0.0 +wasmer==1.1.0 +wasmer_compiler_cranelift==1.1.0 wasmtime==0.36.0 From e7b72b6a6beb13ea7dda56ddf79076443398e744 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Thu, 26 May 2022 15:52:49 +0200 Subject: [PATCH 26/73] Started on our own AST This will help with optimizing code and generating WASM code --- py2wasm/ourlang.py | 713 ++++++++++++++++++++++++++++++++++ py2wasm/utils.py | 12 +- tests/integration/__init__.py | 3 + 3 files changed, 727 insertions(+), 1 deletion(-) create mode 100644 py2wasm/ourlang.py diff --git a/py2wasm/ourlang.py b/py2wasm/ourlang.py new file mode 100644 index 0000000..2898cc2 --- /dev/null +++ b/py2wasm/ourlang.py @@ -0,0 +1,713 @@ +""" +Contains the syntax tree for ourlang +""" +from typing import Any, Dict, List, NoReturn, Union, Tuple + +import ast + +class OurType: + """ + Type base class + """ + __slots__ = () + + def render(self) -> str: + """ + Renders the type back to source code format + + This'll look like Python code. + """ + raise NotImplementedError(self, 'render') + + def alloc_size(self) -> int: + """ + When allocating this type in memory, how many bytes do we need to reserve? + """ + raise NotImplementedError(self, 'alloc_size') + +class OurTypeNone(OurType): + """ + The None (or Void) type + """ + __slots__ = () + + def render(self) -> str: + return 'None' + +class OurTypeInt32(OurType): + """ + The Integer type, signed and 32 bits wide + """ + __slots__ = () + + def render(self) -> str: + return 'i32' + + def alloc_size(self) -> int: + return 4 + +class OurTypeInt64(OurType): + """ + The Integer type, signed and 64 bits wide + """ + __slots__ = () + + def render(self) -> str: + return 'i64' + +class OurTypeFloat32(OurType): + """ + The Float type, 32 bits wide + """ + __slots__ = () + + def render(self) -> str: + return 'f32' + +class OurTypeFloat64(OurType): + """ + The Float type, 64 bits wide + """ + __slots__ = () + + def render(self) -> str: + return 'f64' + +class Expression: + """ + An expression within a statement + """ + __slots__ = () + + def render(self) -> str: + """ + Renders the expression back to source code format + + This'll look like Python code. + """ + raise NotImplementedError(self, 'render') + +class Constant(Expression): + """ + An constant value expression within a statement + """ + __slots__ = ('type', ) + + type: OurType + + def __init__(self, type_: OurType) -> None: + self.type = type_ + +class ConstantInt32(Constant): + """ + An Int32 constant value expression within a statement + """ + __slots__ = ('value', ) + + value: int + + def __init__(self, type_: OurTypeInt32, value: int) -> None: + super().__init__(type_) + self.value = value + + def render(self) -> str: + return str(self.value) + +class ConstantInt64(Constant): + """ + An Int64 constant value expression within a statement + """ + __slots__ = ('value', ) + + value: int + + def __init__(self, type_: OurTypeInt64, value: int) -> None: + super().__init__(type_) + self.value = value + + def render(self) -> str: + return str(self.value) + +class ConstantFloat32(Constant): + """ + An Float32 constant value expression within a statement + """ + __slots__ = ('value', ) + + value: float + + def __init__(self, type_: OurTypeFloat32, value: float) -> None: + super().__init__(type_) + self.value = value + + def render(self) -> str: + return str(self.value) + +class ConstantFloat64(Constant): + """ + An Float64 constant value expression within a statement + """ + __slots__ = ('value', ) + + value: float + + def __init__(self, type_: OurTypeFloat64, value: float) -> None: + super().__init__(type_) + self.value = value + + def render(self) -> str: + return str(self.value) + +class VariableReference(Expression): + """ + An variable reference expression within a statement + """ + __slots__ = ('type', 'name', ) + + type: OurType + name: str + + def __init__(self, type_: OurType, name: str) -> None: + self.type = type_ + self.name = name + + def render(self) -> str: + return str(self.name) + +class BinaryOp(Expression): + """ + A binary operator expression within a statement + """ + __slots__ = ('operator', 'left', 'right', ) + + operator: str + left: Expression + right: Expression + + def __init__(self, operator: str, left: Expression, right: Expression) -> None: + self.operator = operator + self.left = left + self.right = right + + def render(self) -> str: + return f'{self.left.render()} {self.operator} {self.right.render()}' + +class UnaryOp(Expression): + """ + A unary operator expression within a statement + """ + __slots__ = ('operator', 'right', ) + + operator: str + right: Expression + + def __init__(self, operator: str, right: Expression) -> None: + self.operator = operator + self.right = right + + def render(self) -> str: + return f'{self.operator}{self.right.render()}' + +class FunctionCall(Expression): + """ + A function call expression within a statement + """ + __slots__ = ('function', 'arguments', ) + + function: 'Function' + arguments: List[Expression] + + def __init__(self, function: 'Function') -> None: + self.function = function + self.arguments = [] + + def render(self) -> str: + args = ', '.join( + arg.render() + for arg in self.arguments + ) + + return f'{self.function.name}({args})' + +class Statement: + """ + A statement within a function + """ + __slots__ = () + + def render(self) -> List[str]: + """ + Renders the type back to source code format + + This'll look like Python code. + """ + raise NotImplementedError(self, 'render') + +class StatementReturn(Statement): + """ + A return statement within a function + """ + __slots__ = ('value', ) + + def __init__(self, value: Expression) -> None: + self.value = value + + def render(self) -> List[str]: + """ + Renders the type back to source code format + + This'll look like Python code. + """ + return [f'return {self.value.render()}'] + +class StatementIf(Statement): + """ + An if statement within a function + """ + __slots__ = ('test', 'statements', 'else_statements', ) + + test: Expression + statements: List[Statement] + else_statements: List[Statement] + + def __init__(self, test: Expression) -> None: + self.test = test + self.statements = [] + self.else_statements = [] + + def render(self) -> List[str]: + """ + Renders the type back to source code format + + This'll look like Python code. + """ + result = [f'if {self.test.render()}:'] + + for stmt in self.statements: + result.extend( + f' {line}' if line else '' + for line in stmt.render() + ) + + result.append('') + + return result + +class StatementPass(Statement): + """ + A pass statement + """ + __slots__ = () + + def render(self) -> List[str]: + return ['pass'] + +class Function: + """ + A function processes input and produces output + """ + __slots__ = ('name', 'lineno', 'exported', 'statements', 'returns', 'posonlyargs', ) + + name: str + lineno: int + exported: bool + statements: List[Statement] + returns: OurType + posonlyargs: List[Tuple[str, OurType]] + + def __init__(self, name: str, lineno: int) -> None: + self.name = name + self.lineno = lineno + self.exported = False + self.statements = [] + self.returns = OurTypeNone() + self.posonlyargs = [] + + def render(self) -> str: + """ + Renders the function back to source code format + + This'll look like Python code. + """ + result = '' + if self.exported: + result += '@exported\n' + + args = ', '.join( + f'{x}: {y.render()}' + for x, y in self.posonlyargs + ) + + result += f'def {self.name}({args}) -> {self.returns.render()}:\n' + for stmt in self.statements: + for line in stmt.render(): + result += f' {line}\n' if line else '\n' + return result + +class StructMember: + """ + Represents a struct member + """ + def __init__(self, name: str, type_: OurType, offset: int) -> None: + self.name = name + self.type = type_ + self.offset = offset + +class Struct: + """ + A struct has named properties + """ + __slots__ = ('name', 'lineno', 'members', ) + + name: str + lineno: int + members: List[StructMember] + + def __init__(self, name: str, lineno: int) -> None: + self.name = name + self.lineno = lineno + self.members = [] + + def render(self) -> str: + """ + Renders the function back to source code format + + This'll look like Python code. + """ + return '?' + +class Module: + """ + A module is a file and consists of functions + """ + __slots__ = ('types', 'functions', 'structs', ) + + types: Dict[str, OurType] + functions: Dict[str, Function] + structs: Dict[str, Struct] + + def __init__(self) -> None: + self.types = { + 'i32': OurTypeInt32(), + 'i64': OurTypeInt64(), + 'f32': OurTypeFloat32(), + 'f64': OurTypeFloat64(), + } + self.functions = {} + self.structs = {} + + def render(self) -> str: + """ + Renders the module back to source code format + + This'll look like Python code. + """ + result = '' + for function in self.functions.values(): + if result: + result += '\n' + result += function.render() + return result + +class StaticError(Exception): + """ + An error found during static analysis + """ + +OurLocals = Dict[str, OurType] + +class OurVisitor: + """ + Class to visit a Python syntax tree and create an ourlang syntax tree + """ + + # pylint: disable=C0103,C0116,C0301,R0201,R0912 + + def __init__(self) -> None: + pass + + def visit_Module(self, node: ast.Module) -> Module: + module = Module() + + _not_implemented(not node.type_ignores, 'Module.type_ignores') + + for stmt in node.body: + res = self.visit_Module_stmt(module, stmt) + + if isinstance(res, Function): + if res.name in module.functions: + raise StaticError( + f'{res.name} already defined on line {module.functions[res.name].lineno}' + ) + + module.functions[res.name] = res + + if isinstance(res, Struct): + if res.name in module.structs: + raise StaticError( + f'{res.name} already defined on line {module.structs[res.name].lineno}' + ) + + module.structs[res.name] = res + + return module + + def visit_Module_stmt(self, module: Module, node: ast.stmt) -> Union[Function, Struct]: + if isinstance(node, ast.FunctionDef): + return self.visit_Module_FunctionDef(module, node) + + if isinstance(node, ast.ClassDef): + return self.visit_Module_ClassDef(module, node) + + raise NotImplementedError(f'{node} on Module') + + def visit_Module_FunctionDef(self, module: Module, node: ast.FunctionDef) -> Function: + function = Function(node.name, node.lineno) + + _not_implemented(not node.args.posonlyargs, 'FunctionDef.args.posonlyargs') + + for arg in node.args.args: + if not arg.annotation: + _raise_static_error(node, 'Type is required') + + function.posonlyargs.append(( + arg.arg, + self.visit_type(module, arg.annotation), + )) + + _not_implemented(not node.args.vararg, 'FunctionDef.args.vararg') + _not_implemented(not node.args.kwonlyargs, 'FunctionDef.args.kwonlyargs') + _not_implemented(not node.args.kw_defaults, 'FunctionDef.args.kw_defaults') + _not_implemented(not node.args.kwarg, 'FunctionDef.args.kwarg') + _not_implemented(not node.args.defaults, 'FunctionDef.args.defaults') + + # Do stmts at the end so we have the return value + + for decorator in node.decorator_list: + if not isinstance(decorator, ast.Name): + _raise_static_error(decorator, 'Function decorators must be string') + if not isinstance(decorator.ctx, ast.Load): + _raise_static_error(decorator, 'Must be load context') + _not_implemented(decorator.id != 'exports', 'Custom decorators') + function.exported = True + + if node.returns: + function.returns = self.visit_type(module, node.returns) + + _not_implemented(not node.type_comment, 'FunctionDef.type_comment') + + # Deferred parsing + + our_locals = dict(function.posonlyargs) + + for stmt in node.body: + function.statements.append( + self.visit_Module_FunctionDef_stmt(module, function, our_locals, stmt) + ) + + return function + + def visit_Module_ClassDef(self, module: Module, node: ast.ClassDef) -> Struct: + struct = Struct(node.name, node.lineno) + + _not_implemented(not node.bases, 'ClassDef.bases') + _not_implemented(not node.keywords, 'ClassDef.keywords') + _not_implemented(not node.decorator_list, 'ClassDef.decorator_list') + + offset = 0 + + for stmt in node.body: + if not isinstance(stmt, ast.AnnAssign): + raise NotImplementedError(f'Class with {stmt} nodes') + + if not isinstance(stmt.target, ast.Name): + raise NotImplementedError('Class with default values') + + if not stmt.value is None: + raise NotImplementedError('Class with default values') + + if stmt.simple != 1: + raise NotImplementedError('Class with non-simple arguments') + + member = StructMember(stmt.target.id, self.visit_type(module, stmt.annotation), offset) + + struct.members.append(member) + offset += member.type.alloc_size() + + return struct + + def visit_Module_FunctionDef_stmt(self, module: Module, function: Function, our_locals: OurLocals, node: ast.stmt) -> Statement: + if isinstance(node, ast.Return): + if node.value is None: + # TODO: Implement methods without return values + _raise_static_error(node, 'Return must have an argument') + + return StatementReturn( + self.visit_Module_FunctionDef_expr(module, function, our_locals, function.returns, node.value) + ) + + if isinstance(node, ast.If): + result = StatementIf( + self.visit_Module_FunctionDef_expr(module, function, our_locals, function.returns, node.test) + ) + + for stmt in node.body: + result.statements.append( + self.visit_Module_FunctionDef_stmt(module, function, our_locals, stmt) + ) + + for stmt in node.body: + result.else_statements.append( + self.visit_Module_FunctionDef_stmt(module, function, our_locals, stmt) + ) + + return result + + raise NotImplementedError(f'{node} as stmt in FunctionDef') + + def visit_Module_FunctionDef_expr(self, module: Module, function: Function, our_locals: OurLocals, exp_type: OurType, node: ast.expr) -> Expression: + if isinstance(node, ast.BinOp): + if isinstance(node.op, ast.Add): + operator = '+' + elif isinstance(node.op, ast.Sub): + operator = '-' + else: + raise NotImplementedError(f'Operator {node.op}') + + # Assume the type doesn't change when descending into a binary operator + # e.g. you can do `"hello" * 3` with the code below (yet) + + return BinaryOp( + operator, + self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.left), + self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.right), + ) + + if isinstance(node, ast.UnaryOp): + if isinstance(node.op, ast.UAdd): + operator = '+' + elif isinstance(node.op, ast.USub): + operator = '-' + else: + raise NotImplementedError(f'Operator {node.op}') + + return UnaryOp( + operator, + self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.operand), + ) + + if isinstance(node, ast.Compare): + if 1 < len(node.ops): + raise NotImplementedError('Multiple operators') + + if isinstance(node.ops[0], ast.Gt): + operator = '>' + else: + raise NotImplementedError(f'Operator {node.ops}') + + # Assume the type doesn't change when descending into a binary operator + # e.g. you can do `"hello" * 3` with the code below (yet) + + return BinaryOp( + operator, + self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.left), + self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.comparators[0]), + ) + + if isinstance(node, ast.Call): + if node.keywords: + _raise_static_error(node, 'Keyword calling not supported') # Yet? + + if not isinstance(node.func, ast.Name): + raise NotImplementedError(f'Calling methods that are not a name {node.func}') + if not isinstance(node.func.ctx, ast.Load): + _raise_static_error(node, 'Must be load context') + + if node.func.id not in module.functions: + _raise_static_error(node, 'Call to undefined function') + + func = module.functions[node.func.id] + if func.returns != exp_type: + _raise_static_error(node, f'Function does not return {exp_type.render()}') + + result = FunctionCall(func) + result.arguments.extend( + self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, expr) + for expr in node.args + ) + return result + + if isinstance(node, ast.Constant): + return self.visit_Module_FunctionDef_Constant( + module, function, exp_type, node, + ) + + if isinstance(node, ast.Name): + if not isinstance(node.ctx, ast.Load): + _raise_static_error(node, 'Must be load context') + + if node.id in our_locals: + return VariableReference(our_locals[node.id], node.id) + + raise NotImplementedError(f'{node} as expr in FunctionDef') + + def visit_Module_FunctionDef_Constant(self, module: Module, function: Function, exp_type: OurType, node: ast.Constant) -> Expression: + del module + del function + + _not_implemented(node.kind is None, 'Constant.kind') + + if isinstance(exp_type, OurTypeInt32): + if not isinstance(node.value, int): + _raise_static_error(node, 'Expected integer value') + + # FIXME: Range check + + return ConstantInt32(exp_type, node.value) + + if isinstance(exp_type, OurTypeInt64): + if not isinstance(node.value, int): + _raise_static_error(node, 'Expected integer value') + + # FIXME: Range check + + return ConstantInt64(exp_type, node.value) + + if isinstance(exp_type, OurTypeFloat32): + if not isinstance(node.value, float): + _raise_static_error(node, 'Expected float value') + + # FIXME: Range check + + return ConstantFloat32(exp_type, node.value) + + if isinstance(exp_type, OurTypeFloat64): + if not isinstance(node.value, float): + _raise_static_error(node, 'Expected float value') + + # FIXME: Range check + + return ConstantFloat64(exp_type, node.value) + + raise NotImplementedError(f'{node} as const for type {exp_type.render()}') + + def visit_type(self, module: Module, node: ast.expr) -> OurType: + if isinstance(node, ast.Name): + if not isinstance(node.ctx, ast.Load): + _raise_static_error(node, 'Must be load context') + + if node.id not in module.types: + _raise_static_error(node, 'Unrecognized type') + + return module.types[node.id] + + raise NotImplementedError(f'{node} as type') + +def _not_implemented(check: Any, msg: str) -> None: + if not check: + raise NotImplementedError(msg) + +def _raise_static_error(node: Union[ast.mod, ast.stmt, ast.expr], msg: str) -> NoReturn: + raise StaticError( + f'Static error on line {node.lineno}: {msg}' + ) diff --git a/py2wasm/utils.py b/py2wasm/utils.py index c1e9b72..e93d0b5 100644 --- a/py2wasm/utils.py +++ b/py2wasm/utils.py @@ -4,7 +4,8 @@ Utility functions import ast -from py2wasm.python import Visitor +from .ourlang import OurVisitor, Module +from .python import Visitor def process(source: str, input_name: str) -> str: """ @@ -16,3 +17,12 @@ def process(source: str, input_name: str) -> str: module = visitor.visit_Module(res) return module.generate() + +def our_process(source: str, input_name: str) -> Module: + """ + Processes the python code into web assembly code + """ + res = ast.parse(source, input_name) + + our_visitor = OurVisitor() + return our_visitor.visit_Module(res) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index e69de29..e6b179c 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -0,0 +1,3 @@ +import pytest + +pytest.register_assert_rewrite('tests.integration.helpers') From 658e442df2a216e8e2ef15c95f9d77cab2099bb8 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Mon, 6 Jun 2022 12:18:09 +0200 Subject: [PATCH 27/73] - Tuple => () - All tests are now parsed by our own AST --- py2wasm/ourlang.py | 360 +++++++++++++++++++++++++++---- py2wasm/python.py | 6 +- tests/integration/helpers.py | 5 +- tests/integration/test_simple.py | 21 +- 4 files changed, 328 insertions(+), 64 deletions(-) diff --git a/py2wasm/ourlang.py b/py2wasm/ourlang.py index 2898cc2..6550f1c 100644 --- a/py2wasm/ourlang.py +++ b/py2wasm/ourlang.py @@ -1,7 +1,7 @@ """ Contains the syntax tree for ourlang """ -from typing import Any, Dict, List, NoReturn, Union, Tuple +from typing import Any, Dict, List, Optional, NoReturn, Union, Tuple import ast @@ -73,6 +73,21 @@ class OurTypeFloat64(OurType): def render(self) -> str: return 'f64' +class OurTypeTuple(OurType): + """ + The tuple type, 64 bits wide + """ + __slots__ = ('members', ) + + members: List[OurType] + + def __init__(self) -> None: + self.members = [] + + def render(self) -> str: + mems = ', '.join(x.render() for x in self.members) + return f'({mems}, )' + class Expression: """ An expression within a statement @@ -227,8 +242,62 @@ class FunctionCall(Expression): for arg in self.arguments ) + if isinstance(self.function, StructConstructor): + return f'{self.function.struct.name}({args})' + return f'{self.function.name}({args})' +class AccessStructMember(Expression): + """ + Access a struct member for reading of writing + """ + __slots__ = ('varref', 'member', ) + + varref: VariableReference + member: 'StructMember' + + def __init__(self, varref: VariableReference, member: 'StructMember') -> None: + self.varref = varref + self.member = member + + def render(self) -> str: + return f'{self.varref.render()}.{self.member.name}' + +class AccessTupleMember(Expression): + """ + Access a tuple member for reading of writing + """ + __slots__ = ('varref', 'member_idx', 'member_type', ) + + varref: VariableReference + member_idx: int + member_type: 'OurType' + + def __init__(self, varref: VariableReference, member_idx: int, member_type: 'OurType') -> None: + self.varref = varref + self.member_idx = member_idx + self.member_type = member_type + + def render(self) -> str: + return f'{self.varref.render()}[{self.member_idx}]' + +class TupleCreation(Expression): + """ + Create a tuple instance + """ + __slots__ = ('type', 'members', ) + + type: OurTypeTuple + members: List[Expression] + + def __init__(self, type_: OurTypeTuple) -> None: + self.type = type_ + self.members = [] + + def render(self) -> str: + mems = ', '. join(x.render() for x in self.members) + return f'({mems}, )' + class Statement: """ A statement within a function @@ -306,11 +375,12 @@ class Function: """ A function processes input and produces output """ - __slots__ = ('name', 'lineno', 'exported', 'statements', 'returns', 'posonlyargs', ) + __slots__ = ('name', 'lineno', 'exported', 'buildin', 'statements', 'returns', 'posonlyargs', ) name: str lineno: int exported: bool + buildin: bool statements: List[Statement] returns: OurType posonlyargs: List[Tuple[str, OurType]] @@ -319,6 +389,7 @@ class Function: self.name = name self.lineno = lineno self.exported = False + self.buildin = False self.statements = [] self.returns = OurTypeNone() self.posonlyargs = [] @@ -344,6 +415,24 @@ class Function: result += f' {line}\n' if line else '\n' return result +class StructConstructor(Function): + """ + The constructor method for a struct + """ + __slots__ = ('struct', ) + + struct: 'Struct' + + def __init__(self, struct: 'Struct') -> None: + super().__init__(f'@{struct.name}@__init___@', -1) + + self.returns = struct + + for mem in struct.members: + self.posonlyargs.append((mem.name, mem.type, )) + + self.struct = struct + class StructMember: """ Represents a struct member @@ -353,7 +442,7 @@ class StructMember: self.type = type_ self.offset = offset -class Struct: +class Struct(OurType): """ A struct has named properties """ @@ -368,13 +457,35 @@ class Struct: self.lineno = lineno self.members = [] + def get_member(self, name: str) -> Optional[StructMember]: + """ + Returns a member by name + """ + for mem in self.members: + if mem.name == name: + return mem + + return None + def render(self) -> str: """ - Renders the function back to source code format + Renders the type back to source code format This'll look like Python code. """ - return '?' + return self.name + + def render_definition(self) -> str: + """ + Renders the definition back to source code format + + This'll look like Python code. + """ + result = f'class {self.name}:\n' + for mem in self.members: + result += f' {mem.name}: {mem.type.render()}\n' + + return result class Module: """ @@ -396,6 +507,14 @@ class Module: self.functions = {} self.structs = {} + # sqrt is guaranteed by wasm, so we should provide it + # ATM it's a 32 bit variant, but that might change. + sqrt = Function('sqrt', -2) + sqrt.buildin = True + sqrt.returns = self.types['f32'] + sqrt.posonlyargs = [('@', self.types['f32'], )] + self.functions[sqrt.name] = sqrt + def render(self) -> str: """ Renders the module back to source code format @@ -403,10 +522,21 @@ class Module: This'll look like Python code. """ result = '' + + for struct in self.structs.values(): + if result: + result += '\n' + result += struct.render_definition() + for function in self.functions.values(): + if function.lineno < 0: + # Buildin (-2) or auto generated (-1) + continue + if result: result += '\n' result += function.render() + return result class StaticError(Exception): @@ -431,8 +561,10 @@ class OurVisitor: _not_implemented(not node.type_ignores, 'Module.type_ignores') + # Second pass for the types + for stmt in node.body: - res = self.visit_Module_stmt(module, stmt) + res = self.pre_visit_Module_stmt(module, stmt) if isinstance(res, Function): if res.name in module.functions: @@ -449,19 +581,26 @@ class OurVisitor: ) module.structs[res.name] = res + constructor = StructConstructor(res) + module.functions[constructor.name] = constructor + + # Second pass for the function bodies + + for stmt in node.body: + self.visit_Module_stmt(module, stmt) return module - def visit_Module_stmt(self, module: Module, node: ast.stmt) -> Union[Function, Struct]: + def pre_visit_Module_stmt(self, module: Module, node: ast.stmt) -> Union[Function, Struct]: if isinstance(node, ast.FunctionDef): - return self.visit_Module_FunctionDef(module, node) + return self.pre_visit_Module_FunctionDef(module, node) if isinstance(node, ast.ClassDef): - return self.visit_Module_ClassDef(module, node) + return self.pre_visit_Module_ClassDef(module, node) raise NotImplementedError(f'{node} on Module') - def visit_Module_FunctionDef(self, module: Module, node: ast.FunctionDef) -> Function: + def pre_visit_Module_FunctionDef(self, module: Module, node: ast.FunctionDef) -> Function: function = Function(node.name, node.lineno) _not_implemented(not node.args.posonlyargs, 'FunctionDef.args.posonlyargs') @@ -496,18 +635,9 @@ class OurVisitor: _not_implemented(not node.type_comment, 'FunctionDef.type_comment') - # Deferred parsing - - our_locals = dict(function.posonlyargs) - - for stmt in node.body: - function.statements.append( - self.visit_Module_FunctionDef_stmt(module, function, our_locals, stmt) - ) - return function - def visit_Module_ClassDef(self, module: Module, node: ast.ClassDef) -> Struct: + def pre_visit_Module_ClassDef(self, module: Module, node: ast.ClassDef) -> Struct: struct = Struct(node.name, node.lineno) _not_implemented(not node.bases, 'ClassDef.bases') @@ -536,6 +666,26 @@ class OurVisitor: return struct + def visit_Module_stmt(self, module: Module, node: ast.stmt) -> None: + if isinstance(node, ast.FunctionDef): + self.visit_Module_FunctionDef(module, node) + return + + if isinstance(node, ast.ClassDef): + return + + raise NotImplementedError(f'{node} on Module') + + def visit_Module_FunctionDef(self, module: Module, node: ast.FunctionDef) -> None: + function = module.functions[node.name] + + our_locals = dict(function.posonlyargs) + + for stmt in node.body: + function.statements.append( + self.visit_Module_FunctionDef_stmt(module, function, our_locals, stmt) + ) + def visit_Module_FunctionDef_stmt(self, module: Module, function: Function, our_locals: OurLocals, node: ast.stmt) -> Statement: if isinstance(node, ast.Return): if node.value is None: @@ -571,6 +721,8 @@ class OurVisitor: operator = '+' elif isinstance(node.op, ast.Sub): operator = '-' + elif isinstance(node.op, ast.Mult): + operator = '*' else: raise NotImplementedError(f'Operator {node.op}') @@ -602,6 +754,10 @@ class OurVisitor: if isinstance(node.ops[0], ast.Gt): operator = '>' + elif isinstance(node.ops[0], ast.Eq): + operator = '==' + elif isinstance(node.ops[0], ast.Lt): + operator = '<' else: raise NotImplementedError(f'Operator {node.ops}') @@ -615,33 +771,23 @@ class OurVisitor: ) if isinstance(node, ast.Call): - if node.keywords: - _raise_static_error(node, 'Keyword calling not supported') # Yet? - - if not isinstance(node.func, ast.Name): - raise NotImplementedError(f'Calling methods that are not a name {node.func}') - if not isinstance(node.func.ctx, ast.Load): - _raise_static_error(node, 'Must be load context') - - if node.func.id not in module.functions: - _raise_static_error(node, 'Call to undefined function') - - func = module.functions[node.func.id] - if func.returns != exp_type: - _raise_static_error(node, f'Function does not return {exp_type.render()}') - - result = FunctionCall(func) - result.arguments.extend( - self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, expr) - for expr in node.args - ) - return result + return self.visit_Module_FunctionDef_Call(module, function, our_locals, exp_type, node) if isinstance(node, ast.Constant): return self.visit_Module_FunctionDef_Constant( module, function, exp_type, node, ) + if isinstance(node, ast.Attribute): + return self.visit_Module_FunctionDef_Attribute( + module, function, our_locals, exp_type, node, + ) + + if isinstance(node, ast.Subscript): + return self.visit_Module_FunctionDef_Subscript( + module, function, our_locals, exp_type, node, + ) + if isinstance(node, ast.Name): if not isinstance(node.ctx, ast.Load): _raise_static_error(node, 'Must be load context') @@ -649,8 +795,120 @@ class OurVisitor: if node.id in our_locals: return VariableReference(our_locals[node.id], node.id) + if isinstance(node, ast.Tuple): + if not isinstance(node.ctx, ast.Load): + _raise_static_error(node, 'Must be load context') + + if not isinstance(exp_type, OurTypeTuple): + _raise_static_error(node, f'Expression is expecting a {exp_type.render()}, not a tuple') + + if len(exp_type.members) != len(node.elts): + _raise_static_error(node, f'Expression is expecting a tuple of size {len(exp_type.members)}, but {len(node.elts)} are given') + + result = TupleCreation(exp_type) + result.members = [ + self.visit_Module_FunctionDef_expr(module, function, our_locals, arg_type, arg_node) + for arg_node, arg_type in zip(node.elts, exp_type.members) + ] + return result + raise NotImplementedError(f'{node} as expr in FunctionDef') + def visit_Module_FunctionDef_Call(self, module: Module, function: Function, our_locals: OurLocals, exp_type: OurType, node: ast.Call) -> FunctionCall: + if node.keywords: + _raise_static_error(node, 'Keyword calling not supported') # Yet? + + if not isinstance(node.func, ast.Name): + raise NotImplementedError(f'Calling methods that are not a name {node.func}') + if not isinstance(node.func.ctx, ast.Load): + _raise_static_error(node, 'Must be load context') + + if node.func.id in module.structs: + struct = module.structs[node.func.id] + struct_constructor = StructConstructor(struct) + + func = module.functions[struct_constructor.name] + else: + if node.func.id not in module.functions: + _raise_static_error(node, 'Call to undefined function') + + func = module.functions[node.func.id] + + if func.returns != exp_type: + _raise_static_error(node, f'Function {node.func.id} does not return {exp_type.render()}') + + if len(func.posonlyargs) != len(node.args): + _raise_static_error(node, f'Function {node.func.id} requires {len(func.posonlyargs)} arguments but {len(node.args)} are given') + + result = FunctionCall(func) + result.arguments.extend( + self.visit_Module_FunctionDef_expr(module, function, our_locals, arg_type, arg_expr) + for arg_expr, (_, arg_type) in zip(node.args, func.posonlyargs) + ) + return result + + def visit_Module_FunctionDef_Attribute(self, module: Module, function: Function, our_locals: OurLocals, exp_type: OurType, node: ast.Attribute) -> Expression: + if not isinstance(node.value, ast.Name): + _raise_static_error(node, 'Must reference a name') + + if not isinstance(node.ctx, ast.Load): + _raise_static_error(node, 'Must be load context') + + if not node.value.id in our_locals: + _raise_static_error(node, f'Undefined variable {node.value.id}') + + node_typ = our_locals[node.value.id] + if not isinstance(node_typ, Struct): + _raise_static_error(node, f'Cannot take attribute of non-struct {node.value.id}') + + member = node_typ.get_member(node.attr) + if member is None: + _raise_static_error(node, f'{node_typ.name} has no attribute {node.attr}') + + if exp_type != member.type: + _raise_static_error(node, f'Expected {exp_type.render()}, got {member.type.render()} instead') + + return AccessStructMember( + VariableReference(node_typ, node.value.id), + member, + ) + + def visit_Module_FunctionDef_Subscript(self, module: Module, function: Function, our_locals: OurLocals, exp_type: OurType, node: ast.Subscript) -> Expression: + if not isinstance(node.value, ast.Name): + _raise_static_error(node, 'Must reference a name') + + if not isinstance(node.slice, ast.Index): + _raise_static_error(node, 'Must subscript using an index') + + if not isinstance(node.slice.value, ast.Constant): + _raise_static_error(node, 'Must subscript using a constant index') # FIXME: Implement variable indexes + + if not isinstance(node.slice.value.value, int): + _raise_static_error(node, 'Must subscript using a constant integer index') + + if not isinstance(node.ctx, ast.Load): + _raise_static_error(node, 'Must be load context') + + if not node.value.id in our_locals: + _raise_static_error(node, f'Undefined variable {node.value.id}') + + node_typ = our_locals[node.value.id] + if not isinstance(node_typ, OurTypeTuple): + _raise_static_error(node, f'Cannot take index of non-tuple {node.value.id}') + + if len(node_typ.members) <= node.slice.value.value: + _raise_static_error(node, f'Index {node.slice.value.value} out of bounds for tuple {node.value.id}') + + member = node_typ.members[node.slice.value.value] + if exp_type != member: + _raise_static_error(node, f'Expected {exp_type.render()}, got {member.render()} instead') + + return AccessTupleMember( + VariableReference(node_typ, node.value.id), + node.slice.value.value, + member, + ) + def visit_Module_FunctionDef_Constant(self, module: Module, function: Function, exp_type: OurType, node: ast.Constant) -> Expression: del module del function @@ -696,10 +954,24 @@ class OurVisitor: if not isinstance(node.ctx, ast.Load): _raise_static_error(node, 'Must be load context') - if node.id not in module.types: - _raise_static_error(node, 'Unrecognized type') + if node.id in module.types: + return module.types[node.id] - return module.types[node.id] + if node.id in module.structs: + return module.structs[node.id] + + _raise_static_error(node, f'Unrecognized type {node.id}') + + if isinstance(node, ast.Tuple): + if not isinstance(node.ctx, ast.Load): + _raise_static_error(node, 'Must be load context') + + result = OurTypeTuple() + result.members = [ + self.visit_type(module, elt) + for elt in node.elts + ] + return result raise NotImplementedError(f'{node} as type') diff --git a/py2wasm/python.py b/py2wasm/python.py index 4867061..529749f 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -41,14 +41,12 @@ class Visitor: if isinstance(name, ast.expr): if isinstance(name, ast.Name): name = name.id - elif isinstance(name, ast.Subscript) and isinstance(name.value, ast.Name) and name.value.id == 'Tuple': + elif isinstance(name, ast.Tuple): assert isinstance(name.ctx, ast.Load) - assert isinstance(name.slice, ast.Index) - assert isinstance(name.slice.value, ast.Tuple) args: List[wasm.TupleMember] = [] offset = 0 - for name_arg in name.slice.value.elts: + for name_arg in name.elts: arg = wasm.TupleMember( self._get_type(name_arg), offset diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 98f5dad..634612f 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -14,7 +14,7 @@ import wasmer_compiler_cranelift import wasmtime -from py2wasm.utils import process +from py2wasm.utils import our_process, process DASHES = '-' * 16 @@ -69,6 +69,9 @@ class Suite: Returned is an object with the results set """ + our_module = our_process(self.code_py, self.test_name) + assert self.code_py == '\n' + our_module.render() # \n for formatting in tests + code_wat = process(self.code_py, self.test_name) sys.stderr.write(f'{DASHES} Assembly {DASHES}\n') diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index 0703e08..d7ee5f2 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -179,6 +179,7 @@ def testEntry(a: i32) -> i32: assert 1 == result.returned_value @pytest.mark.integration_test +@pytest.mark.skip('Such a return is not how things should be') def test_if_complex(): code_py = """ @exported @@ -254,7 +255,6 @@ def testEntry() -> i32: @pytest.mark.integration_test def test_struct_0(): code_py = """ - class CheckedValue: value: i32 @@ -274,11 +274,10 @@ def helper(cv: CheckedValue) -> i32: @pytest.mark.integration_test def test_struct_1(): code_py = """ - class Rectangle: height: i32 width: i32 - border: i32 # = 5 + border: i32 @exported def testEntry() -> i32: @@ -296,21 +295,17 @@ def helper(shape: Rectangle) -> i32: @pytest.mark.integration_test def test_struct_2(): code_py = """ - class Rectangle: height: i32 width: i32 - border: i32 # = 5 + border: i32 @exported def testEntry() -> i32: return helper(Rectangle(100, 150, 2), Rectangle(200, 90, 3)) def helper(shape1: Rectangle, shape2: Rectangle) -> i32: - return ( - shape1.height + shape1.width + shape1.border - + shape2.height + shape2.width + shape2.border - ) + return shape1.height + shape1.width + shape1.border + shape2.height + shape2.width + shape2.border """ result = Suite(code_py, 'test_call').run_code() @@ -321,12 +316,11 @@ def helper(shape1: Rectangle, shape2: Rectangle) -> i32: @pytest.mark.integration_test def test_tuple_int(): code_py = """ - @exported def testEntry() -> i32: return helper((24, 57, 80, )) -def helper(vector: Tuple[i32, i32, i32]) -> i32: +def helper(vector: (i32, i32, i32, )) -> i32: return vector[0] + vector[1] + vector[2] """ @@ -338,14 +332,11 @@ def helper(vector: Tuple[i32, i32, i32]) -> i32: @pytest.mark.integration_test def test_tuple_float(): code_py = """ - @exported def testEntry() -> f32: return helper((1.0, 2.0, 3.0, )) -def helper(v: Tuple[f32, f32, f32]) -> f32: - # sqrt is guaranteed by wasm, so we can use it - # ATM it's a 32 bit variant, but that might change. +def helper(v: (f32, f32, f32, )) -> f32: return sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]) """ From 8c25227f401df790677c31d44d0aba54ffa91f07 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sun, 19 Jun 2022 15:10:13 +0200 Subject: [PATCH 28/73] Started on compilation, typing changes --- py2wasm/compiler.py | 115 +++++++++++++++++++++++++++++++ py2wasm/ourlang.py | 32 +++++---- py2wasm/python.py | 6 ++ py2wasm/wasm.py | 5 +- tests/integration/test_simple.py | 85 ++++++++++------------- 5 files changed, 180 insertions(+), 63 deletions(-) create mode 100644 py2wasm/compiler.py diff --git a/py2wasm/compiler.py b/py2wasm/compiler.py new file mode 100644 index 0000000..425adb6 --- /dev/null +++ b/py2wasm/compiler.py @@ -0,0 +1,115 @@ +""" +This module contains the code to convert parsed Ourlang into WebAssembly code +""" +from typing import Generator, Tuple + +from . import ourlang +from . import wasm + +Statements = Generator[wasm.Statement, None, None] + +def type_(inp: ourlang.OurType) -> wasm.OurType: + if isinstance(inp, ourlang.OurTypeInt32): + return wasm.OurTypeInt32() + + if isinstance(inp, ourlang.OurTypeInt64): + return wasm.OurTypeInt64() + + if isinstance(inp, ourlang.OurTypeFloat32): + return wasm.OurTypeFloat32() + + if isinstance(inp, ourlang.OurTypeFloat64): + return wasm.OurTypeFloat64() + + raise NotImplementedError(type_, inp) + +OPERATOR_MAP = { + '+': 'add', + '-': 'sub', +} + +def expression(inp: ourlang.Expression) -> Statements: + if isinstance(inp, ourlang.ConstantInt32): + yield wasm.Statement('i32.const', str(inp.value)) + return + + if isinstance(inp, ourlang.ConstantInt64): + yield wasm.Statement('i64.const', str(inp.value)) + return + + if isinstance(inp, ourlang.ConstantFloat32): + yield wasm.Statement('f32.const', str(inp.value)) + return + + if isinstance(inp, ourlang.ConstantFloat64): + yield wasm.Statement('f64.const', str(inp.value)) + return + + if isinstance(inp, ourlang.VariableReference): + yield wasm.Statement('local.get', '${}'.format(inp.name)) + return + + if isinstance(inp, ourlang.BinaryOp): + yield from expression(inp.left) + yield from expression(inp.right) + + if isinstance(inp.type, ourlang.OurTypeInt32): + if operator := OPERATOR_MAP.get(inp.operator, None): + yield wasm.Statement(f'i32.{operator}') + return + if isinstance(inp.type, ourlang.OurTypeInt64): + if operator := OPERATOR_MAP.get(inp.operator, None): + yield wasm.Statement(f'i64.{operator}') + return + if isinstance(inp.type, ourlang.OurTypeFloat32): + if operator := OPERATOR_MAP.get(inp.operator, None): + yield wasm.Statement(f'f32.{operator}') + return + if isinstance(inp.type, ourlang.OurTypeFloat64): + if operator := OPERATOR_MAP.get(inp.operator, None): + yield wasm.Statement(f'f64.{operator}') + return + + raise NotImplementedError(expression, inp.type, inp.operator) + + raise NotImplementedError(expression, inp) + +def statement_return(inp: ourlang.StatementReturn) -> Statements: + yield from expression(inp.value) + yield wasm.Statement('return') + +def statement(inp: ourlang.Statement) -> Statements: + if isinstance(inp, ourlang.StatementReturn): + yield from statement_return(inp) + return + + raise NotImplementedError(statement, inp) + +def function_argument(inp: Tuple[str, ourlang.OurType]) -> wasm.Param: + return (inp[0], type_(inp[1]), ) + +def function(inp: ourlang.Function) -> wasm.Function: + return wasm.Function( + inp.name, + inp.exported, + [ + function_argument(x) + for x in inp.posonlyargs + ], + [], # TODO + type_(inp.returns), + [ + x + for y in inp.statements + for x in statement(y) + ] + ) + +def module(inp: ourlang.Module) -> wasm.Module: + result = wasm.Module() + + result.functions = [*map( + function, inp.functions.values(), + )] + + return result diff --git a/py2wasm/ourlang.py b/py2wasm/ourlang.py index 6550f1c..b471514 100644 --- a/py2wasm/ourlang.py +++ b/py2wasm/ourlang.py @@ -92,7 +92,12 @@ class Expression: """ An expression within a statement """ - __slots__ = () + __slots__ = ('type', ) + + type: OurType + + def __init__(self, type_: OurType) -> None: + self.type = type_ def render(self) -> str: """ @@ -106,12 +111,7 @@ class Constant(Expression): """ An constant value expression within a statement """ - __slots__ = ('type', ) - - type: OurType - - def __init__(self, type_: OurType) -> None: - self.type = type_ + __slots__ = () class ConstantInt32(Constant): """ @@ -177,13 +177,12 @@ class VariableReference(Expression): """ An variable reference expression within a statement """ - __slots__ = ('type', 'name', ) + __slots__ = ('name', ) - type: OurType name: str def __init__(self, type_: OurType, name: str) -> None: - self.type = type_ + super().__init__(type_) self.name = name def render(self) -> str: @@ -199,7 +198,9 @@ class BinaryOp(Expression): left: Expression right: Expression - def __init__(self, operator: str, left: Expression, right: Expression) -> None: + def __init__(self, type_: OurType, operator: str, left: Expression, right: Expression) -> None: + super().__init__(type_) + self.operator = operator self.left = left self.right = right @@ -509,11 +510,12 @@ class Module: # sqrt is guaranteed by wasm, so we should provide it # ATM it's a 32 bit variant, but that might change. + # TODO: Could be a UnaryOp? sqrt = Function('sqrt', -2) sqrt.buildin = True sqrt.returns = self.types['f32'] sqrt.posonlyargs = [('@', self.types['f32'], )] - self.functions[sqrt.name] = sqrt + # self.functions[sqrt.name] = sqrt def render(self) -> str: """ @@ -730,6 +732,7 @@ class OurVisitor: # e.g. you can do `"hello" * 3` with the code below (yet) return BinaryOp( + exp_type, operator, self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.left), self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.right), @@ -765,6 +768,7 @@ class OurVisitor: # e.g. you can do `"hello" * 3` with the code below (yet) return BinaryOp( + exp_type, operator, self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.left), self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.comparators[0]), @@ -932,7 +936,7 @@ class OurVisitor: return ConstantInt64(exp_type, node.value) if isinstance(exp_type, OurTypeFloat32): - if not isinstance(node.value, float): + if not isinstance(node.value, (float, int, )): _raise_static_error(node, 'Expected float value') # FIXME: Range check @@ -940,7 +944,7 @@ class OurVisitor: return ConstantFloat32(exp_type, node.value) if isinstance(exp_type, OurTypeFloat64): - if not isinstance(node.value, float): + if not isinstance(node.value, (float, int, )): _raise_static_error(node, 'Expected float value') # FIXME: Range check diff --git a/py2wasm/python.py b/py2wasm/python.py index 529749f..91cef95 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -585,6 +585,9 @@ class Visitor: return if isinstance(exp_type, wasm.OurTypeFloat32): + if isinstance(node.value, int): + node.value = float(node.value) + assert isinstance(node.value, float), \ f'Expression expected a float, but received {node.value}' # TODO: Size check? @@ -593,6 +596,9 @@ class Visitor: return if isinstance(exp_type, wasm.OurTypeFloat64): + if isinstance(node.value, int): + node.value = float(node.value) + assert isinstance(node.value, float) # TODO: Size check? diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py index b9b7d85..09609e6 100644 --- a/py2wasm/wasm.py +++ b/py2wasm/wasm.py @@ -1,5 +1,6 @@ """ -Python classes for storing the representation of Web Assembly code +Python classes for storing the representation of Web Assembly code, +and being able to conver it to Web Assembly Text Format """ from typing import Any, Iterable, List, Optional, Tuple, Union @@ -220,7 +221,7 @@ class Function: for nam, typ in self.params: header += f' (param ${nam} {typ.to_wasm()})' - if self.result: + if not isinstance(self.result, OurTypeNone): header += f' (result {self.result.to_wasm()})' for nam, typ in self.locals: diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index d7ee5f2..c1af722 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -2,72 +2,71 @@ import pytest from .helpers import Suite +TYPE_MAP = { + 'i32': int, + 'i64': int, + 'f32': float, + 'f64': float, +} + @pytest.mark.integration_test -def test_return_i32(): - code_py = """ +@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) +def test_return(type_): + code_py = f""" @exported -def testEntry() -> i32: +def testEntry() -> {type_}: return 13 """ result = Suite(code_py, 'test_return').run_code() assert 13 == result.returned_value - assert [] == result.log_int32_list + assert TYPE_MAP[type_] == type(result.returned_value) @pytest.mark.integration_test -def test_return_i64(): - code_py = """ +@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) +def test_addition(type_): + code_py = f""" @exported -def testEntry() -> i64: - return 13 +def testEntry() -> {type_}: + return 10 + 3 """ - result = Suite(code_py, 'test_return').run_code() + result = Suite(code_py, 'test_addition').run_code() assert 13 == result.returned_value - assert [] == result.log_int32_list + assert TYPE_MAP[type_] == type(result.returned_value) @pytest.mark.integration_test -def test_return_f32(): - code_py = """ +@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) +def test_subtraction(type_): + code_py = f""" @exported -def testEntry() -> f32: - return 13.5 +def testEntry() -> {type_}: + return 10 - 3 """ - result = Suite(code_py, 'test_return').run_code() + result = Suite(code_py, 'test_addition').run_code() - assert 13.5 == result.returned_value - assert [] == result.log_int32_list + assert 7 == result.returned_value + assert TYPE_MAP[type_] == type(result.returned_value) @pytest.mark.integration_test -def test_return_f64(): - code_py = """ +@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) +def test_arg(type_): + code_py = f""" @exported -def testEntry() -> f64: - return 13.5 -""" - - result = Suite(code_py, 'test_return').run_code() - - assert 13.5 == result.returned_value - assert [] == result.log_int32_list - -@pytest.mark.integration_test -def test_arg(): - code_py = """ -@exported -def testEntry(a: i32) -> i32: +def testEntry(a: {type_}) -> {type_}: return a """ result = Suite(code_py, 'test_return').run_code(125) assert 125 == result.returned_value - assert [] == result.log_int32_list + assert TYPE_MAP[type_] == type(result.returned_value) @pytest.mark.integration_test +@pytest.mark.skip('Do we want it to work like this?') def test_i32_to_i64(): code_py = """ @exported @@ -81,6 +80,7 @@ def testEntry(a: i32) -> i64: assert [] == result.log_int32_list @pytest.mark.integration_test +@pytest.mark.skip('Do we want it to work like this?') def test_i32_plus_i64(): code_py = """ @exported @@ -94,6 +94,7 @@ def testEntry(a: i32, b: i64) -> i64: assert [] == result.log_int32_list @pytest.mark.integration_test +@pytest.mark.skip('Do we want it to work like this?') def test_f32_to_f64(): code_py = """ @exported @@ -107,6 +108,7 @@ def testEntry(a: f32) -> f64: assert [] == result.log_int32_list @pytest.mark.integration_test +@pytest.mark.skip('Do we want it to work like this?') def test_f32_plus_f64(): code_py = """ @exported @@ -120,6 +122,7 @@ def testEntry(a: f32, b: f64) -> f64: assert [] == result.log_int32_list @pytest.mark.integration_test +@pytest.mark.skip('TODO') def test_uadd(): code_py = """ @exported @@ -133,6 +136,7 @@ def testEntry() -> i32: assert [] == result.log_int32_list @pytest.mark.integration_test +@pytest.mark.skip('TODO') def test_usub(): code_py = """ @exported @@ -145,19 +149,6 @@ def testEntry() -> i32: assert -19 == result.returned_value assert [] == result.log_int32_list -@pytest.mark.integration_test -def test_addition(): - code_py = """ -@exported -def testEntry() -> i32: - return 10 + 3 -""" - - result = Suite(code_py, 'test_addition').run_code() - - assert 13 == result.returned_value - assert [] == result.log_int32_list - @pytest.mark.integration_test def test_if_simple(): code_py = """ From 83b0b705ae9eaa094665280806f0c354d80b01c2 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sun, 19 Jun 2022 15:20:47 +0200 Subject: [PATCH 29/73] If statement, more operators --- py2wasm/compiler.py | 29 +++++++++++++++++++++++++++++ py2wasm/ourlang.py | 2 +- tests/integration/test_simple.py | 16 +++++++--------- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/py2wasm/compiler.py b/py2wasm/compiler.py index 425adb6..72a87a3 100644 --- a/py2wasm/compiler.py +++ b/py2wasm/compiler.py @@ -28,6 +28,13 @@ OPERATOR_MAP = { '-': 'sub', } +I32_OPERATOR_MAP = { # TODO: Introduce UInt32 type + '<': 'lt_s', + '>': 'gt_s', + '<=': 'le_s', + '>=': 'ge_s', +} + def expression(inp: ourlang.Expression) -> Statements: if isinstance(inp, ourlang.ConstantInt32): yield wasm.Statement('i32.const', str(inp.value)) @@ -57,6 +64,9 @@ def expression(inp: ourlang.Expression) -> Statements: if operator := OPERATOR_MAP.get(inp.operator, None): yield wasm.Statement(f'i32.{operator}') return + if operator := I32_OPERATOR_MAP.get(inp.operator, None): + yield wasm.Statement(f'i32.{operator}') + return if isinstance(inp.type, ourlang.OurTypeInt64): if operator := OPERATOR_MAP.get(inp.operator, None): yield wasm.Statement(f'i64.{operator}') @@ -78,11 +88,30 @@ def statement_return(inp: ourlang.StatementReturn) -> Statements: yield from expression(inp.value) yield wasm.Statement('return') +def statement_if(inp: ourlang.StatementIf) -> Statements: + yield from expression(inp.test) + + yield wasm.Statement('if') + + for stat in inp.statements: + yield from statement(stat) + + if inp.else_statements: + yield wasm.Statement('else') + for stat in inp.else_statements: + yield from statement(stat) + + yield wasm.Statement('end') + def statement(inp: ourlang.Statement) -> Statements: if isinstance(inp, ourlang.StatementReturn): yield from statement_return(inp) return + if isinstance(inp, ourlang.StatementIf): + yield from statement_if(inp) + return + raise NotImplementedError(statement, inp) def function_argument(inp: Tuple[str, ourlang.OurType]) -> wasm.Param: diff --git a/py2wasm/ourlang.py b/py2wasm/ourlang.py index b471514..16661c8 100644 --- a/py2wasm/ourlang.py +++ b/py2wasm/ourlang.py @@ -708,7 +708,7 @@ class OurVisitor: self.visit_Module_FunctionDef_stmt(module, function, our_locals, stmt) ) - for stmt in node.body: + for stmt in node.orelse: result.else_statements.append( self.visit_Module_FunctionDef_stmt(module, function, our_locals, stmt) ) diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index c1af722..f54a715 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -150,24 +150,22 @@ def testEntry() -> i32: assert [] == result.log_int32_list @pytest.mark.integration_test -def test_if_simple(): +@pytest.mark.parametrize('inp', [9, 10, 11, 12]) +def test_if_simple(inp): code_py = """ @exported def testEntry(a: i32) -> i32: if a > 10: - return 1 + return 15 - return 0 + return 3 """ + exp_result = 15 if inp > 10 else 3 suite = Suite(code_py, 'test_return') - result = suite.run_code(10) - assert 0 == result.returned_value - assert [] == result.log_int32_list - - result = suite.run_code(11) - assert 1 == result.returned_value + result = suite.run_code(inp) + assert exp_result == result.returned_value @pytest.mark.integration_test @pytest.mark.skip('Such a return is not how things should be') From 9dbdb1173267962a4070263b32e0b13679886b28 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sun, 19 Jun 2022 15:25:58 +0200 Subject: [PATCH 30/73] Simple calls --- py2wasm/compiler.py | 7 +++++++ tests/integration/test_simple.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/py2wasm/compiler.py b/py2wasm/compiler.py index 72a87a3..355e4b4 100644 --- a/py2wasm/compiler.py +++ b/py2wasm/compiler.py @@ -82,6 +82,13 @@ def expression(inp: ourlang.Expression) -> Statements: raise NotImplementedError(expression, inp.type, inp.operator) + if isinstance(inp, ourlang.FunctionCall): + for arg in inp.arguments: + yield from expression(arg) + + yield wasm.Statement('call', '${}'.format(inp.function.name)) + return + raise NotImplementedError(expression, inp) def statement_return(inp: ourlang.StatementReturn) -> Statements: diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index f54a715..97cd31b 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -225,6 +225,23 @@ def helper(left: i32, right: i32) -> i32: assert 7 == result.returned_value assert [] == result.log_int32_list +@pytest.mark.integration_test +@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) +def test_call_with_expression(type_): + code_py = f""" +@exported +def testEntry() -> {type_}: + return helper(10 + 20, 3 + 5) + +def helper(left: {type_}, right: {type_}) -> {type_}: + return left - right +""" + + result = Suite(code_py, 'test_call').run_code() + + assert 22 == result.returned_value + assert TYPE_MAP[type_] == type(result.returned_value) + @pytest.mark.integration_test @pytest.mark.skip('Not yet implemented') def test_assign(): From 453c2865a8a74bdc717e2233f05dc24c5a06113a Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sun, 19 Jun 2022 16:09:06 +0200 Subject: [PATCH 31/73] Structs --- py2wasm/compiler.py | 94 ++++++++++++++++++++++++++++++++++++++++----- py2wasm/ourlang.py | 22 +++++++++-- py2wasm/python.py | 2 +- py2wasm/wasm.py | 17 +++++++- 4 files changed, 120 insertions(+), 15 deletions(-) diff --git a/py2wasm/compiler.py b/py2wasm/compiler.py index 355e4b4..eba158c 100644 --- a/py2wasm/compiler.py +++ b/py2wasm/compiler.py @@ -21,6 +21,11 @@ def type_(inp: ourlang.OurType) -> wasm.OurType: if isinstance(inp, ourlang.OurTypeFloat64): return wasm.OurTypeFloat64() + if isinstance(inp, ourlang.Struct): + # Structs are passed as pointer + # And pointers are i32 + return wasm.OurTypeInt32() + raise NotImplementedError(type_, inp) OPERATOR_MAP = { @@ -89,6 +94,19 @@ def expression(inp: ourlang.Expression) -> Statements: yield wasm.Statement('call', '${}'.format(inp.function.name)) return + if isinstance(inp, ourlang.AccessStructMember): + # FIXME: Properly implement this + # inp.type.render() is also a hack that doesn't really work consistently + if not isinstance(inp.type, ( + ourlang.OurTypeInt32, ourlang.OurTypeFloat32, + ourlang.OurTypeInt64, ourlang.OurTypeFloat64, + )): + raise NotImplementedError(inp, inp.type) + + yield from expression(inp.varref) + yield wasm.Statement(inp.type.render() + '.load', 'offset=' + str(inp.member.offset)) + return + raise NotImplementedError(expression, inp) def statement_return(inp: ourlang.StatementReturn) -> Statements: @@ -125,6 +143,21 @@ def function_argument(inp: Tuple[str, ourlang.OurType]) -> wasm.Param: return (inp[0], type_(inp[1]), ) def function(inp: ourlang.Function) -> wasm.Function: + if isinstance(inp, ourlang.StructConstructor): + statements = [ + *_generate_struct_constructor(inp) + ] + locals_ = [ + ('___new_reference___addr', wasm.OurTypeInt32(), ), + ] + else: + statements = [ + x + for y in inp.statements + for x in statement(y) + ] + locals_ = [] # TODO + return wasm.Function( inp.name, inp.exported, @@ -132,20 +165,63 @@ def function(inp: ourlang.Function) -> wasm.Function: function_argument(x) for x in inp.posonlyargs ], - [], # TODO + locals_, type_(inp.returns), - [ - x - for y in inp.statements - for x in statement(y) - ] + statements ) def module(inp: ourlang.Module) -> wasm.Module: result = wasm.Module() - result.functions = [*map( - function, inp.functions.values(), - )] + result.functions = [ + _generate_allocator(inp), + ] + [ + function(x) + for x in inp.functions.values() + ] return result + +def _generate_allocator(mod: ourlang.Module) -> wasm.Function: + return wasm.Function( + '___new_reference___', + False, + [ + ('alloc_size', type_(mod.types['i32']), ), + ], + [ + ('result', type_(mod.types['i32']), ), + ], + type_(mod.types['i32']), + [ + wasm.Statement('i32.const', '0'), + wasm.Statement('i32.const', '0'), + wasm.Statement('i32.load'), + wasm.Statement('local.tee', '$result', comment='Address for this call'), + wasm.Statement('local.get', '$alloc_size'), + wasm.Statement('i32.add'), + wasm.Statement('i32.store', comment='Address for the next call'), + wasm.Statement('local.get', '$result'), + ], + ) + +def _generate_struct_constructor(inp: ourlang.StructConstructor) -> Statements: + yield wasm.Statement('i32.const', str(inp.struct.alloc_size())) + yield wasm.Statement('call', '$___new_reference___') + + yield wasm.Statement('local.set', '$___new_reference___addr') + + for member in inp.struct.members: + # FIXME: Properly implement this + # inp.type.render() is also a hack that doesn't really work consistently + if not isinstance(member.type, ( + ourlang.OurTypeInt32, ourlang.OurTypeFloat32, + ourlang.OurTypeInt64, ourlang.OurTypeFloat64, + )): + raise NotImplementedError + + yield wasm.Statement('local.get', '$___new_reference___addr') + yield wasm.Statement('local.get', f'${member.name}') + yield wasm.Statement(f'{member.type.render()}.store', 'offset=' + str(member.offset)) + + yield wasm.Statement('local.get', '$___new_reference___addr') diff --git a/py2wasm/ourlang.py b/py2wasm/ourlang.py index 16661c8..0d53280 100644 --- a/py2wasm/ourlang.py +++ b/py2wasm/ourlang.py @@ -217,7 +217,9 @@ class UnaryOp(Expression): operator: str right: Expression - def __init__(self, operator: str, right: Expression) -> None: + def __init__(self, type_: OurType, operator: str, right: Expression) -> None: + super().__init__(type_) + self.operator = operator self.right = right @@ -234,6 +236,8 @@ class FunctionCall(Expression): arguments: List[Expression] def __init__(self, function: 'Function') -> None: + super().__init__(function.returns) + self.function = function self.arguments = [] @@ -258,6 +262,8 @@ class AccessStructMember(Expression): member: 'StructMember' def __init__(self, varref: VariableReference, member: 'StructMember') -> None: + super().__init__(member.type) + self.varref = varref self.member = member @@ -275,6 +281,8 @@ class AccessTupleMember(Expression): member_type: 'OurType' def __init__(self, varref: VariableReference, member_idx: int, member_type: 'OurType') -> None: + super().__init__(member_type) + self.varref = varref self.member_idx = member_idx self.member_type = member_type @@ -286,13 +294,14 @@ class TupleCreation(Expression): """ Create a tuple instance """ - __slots__ = ('type', 'members', ) + __slots__ = ('members', ) type: OurTypeTuple members: List[Expression] def __init__(self, type_: OurTypeTuple) -> None: - self.type = type_ + super().__init__(type_) + self.members = [] def render(self) -> str: @@ -488,6 +497,12 @@ class Struct(OurType): return result + def alloc_size(self) -> int: + return sum( + x.type.alloc_size() + for x in self.members + ) + class Module: """ A module is a file and consists of functions @@ -747,6 +762,7 @@ class OurVisitor: raise NotImplementedError(f'Operator {node.op}') return UnaryOp( + exp_type, operator, self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.operand), ) diff --git a/py2wasm/python.py b/py2wasm/python.py index 91cef95..18d6869 100644 --- a/py2wasm/python.py +++ b/py2wasm/python.py @@ -199,7 +199,7 @@ class Visitor: params.append((arg.arg, self._get_type(arg.annotation), )) locals_: List[wasm.Param] = [ - ('___new_reference___addr', self._get_type('i32')), # For the ___new_reference__ method + ('___new_reference___addr', self._get_type('i32')), # For the ___new_reference___ method ] return wasm.Function(node.name, exported, params, locals_, result, []) diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py index 09609e6..49612cd 100644 --- a/py2wasm/wasm.py +++ b/py2wasm/wasm.py @@ -232,6 +232,18 @@ class Function: '\n '.join(x.generate() for x in self.statements), ) +class ModuleMemory: + def __init__(self, data: bytes = b'') -> None: + self.data = data + + def generate(self) -> str: + return '(memory 1)\n (data (memory 0) (i32.const 0) "{}")\n'.format( + ''.join( + f'\\{x:02x}' + for x in self.data + ) + ) + class Module: """ Represents a Web Assembly module @@ -239,13 +251,14 @@ class Module: def __init__(self) -> None: self.imports: List[Import] = [] self.functions: List[Function] = [] + self.memory = ModuleMemory(b'\x04') # For ___new_reference___ def generate(self) -> str: """ Generates the text version """ - return '(module\n (memory 1)\n (data (memory 0) (i32.const 0) {})\n {}\n {})\n'.format( - '"\\04\\00\\00\\00"', + return '(module\n {}\n {}\n {})\n'.format( + self.memory.generate(), '\n '.join(x.generate() for x in self.imports), '\n '.join(x.generate() for x in self.functions), ) From a480e6069861d6cfe42007ea8547111e199c2b6e Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sun, 19 Jun 2022 16:26:58 +0200 Subject: [PATCH 32/73] Buildin float ops --- py2wasm/compiler.py | 14 ++++++++++++++ py2wasm/ourlang.py | 30 ++++++++++++++++++++---------- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/py2wasm/compiler.py b/py2wasm/compiler.py index eba158c..d04bb2c 100644 --- a/py2wasm/compiler.py +++ b/py2wasm/compiler.py @@ -87,6 +87,20 @@ def expression(inp: ourlang.Expression) -> Statements: raise NotImplementedError(expression, inp.type, inp.operator) + if isinstance(inp, ourlang.UnaryOp): + yield from expression(inp.right) + + if isinstance(inp.type, ourlang.OurTypeFloat32): + if inp.operator in ourlang.WEBASSEMBLY_BUILDIN_FLOAT_OPS: + yield wasm.Statement(f'f32.{inp.operator}') + return + if isinstance(inp.type, ourlang.OurTypeFloat64): + if inp.operator in ourlang.WEBASSEMBLY_BUILDIN_FLOAT_OPS: + yield wasm.Statement(f'f64.{inp.operator}') + return + + raise NotImplementedError(expression, inp.type, inp.operator) + if isinstance(inp, ourlang.FunctionCall): for arg in inp.arguments: yield from expression(arg) diff --git a/py2wasm/ourlang.py b/py2wasm/ourlang.py index 0d53280..1855e21 100644 --- a/py2wasm/ourlang.py +++ b/py2wasm/ourlang.py @@ -5,6 +5,10 @@ from typing import Any, Dict, List, Optional, NoReturn, Union, Tuple import ast +from typing_extensions import Final + +WEBASSEMBLY_BUILDIN_FLOAT_OPS: Final = ('abs', 'sqrt', 'ceil', 'floor', 'trunc', 'nearest', ) + class OurType: """ Type base class @@ -224,6 +228,9 @@ class UnaryOp(Expression): self.right = right def render(self) -> str: + if self.operator in WEBASSEMBLY_BUILDIN_FLOAT_OPS: + return f'{self.operator}({self.right.render()})' + return f'{self.operator}{self.right.render()}' class FunctionCall(Expression): @@ -523,15 +530,6 @@ class Module: self.functions = {} self.structs = {} - # sqrt is guaranteed by wasm, so we should provide it - # ATM it's a 32 bit variant, but that might change. - # TODO: Could be a UnaryOp? - sqrt = Function('sqrt', -2) - sqrt.buildin = True - sqrt.returns = self.types['f32'] - sqrt.posonlyargs = [('@', self.types['f32'], )] - # self.functions[sqrt.name] = sqrt - def render(self) -> str: """ Renders the module back to source code format @@ -834,7 +832,7 @@ class OurVisitor: raise NotImplementedError(f'{node} as expr in FunctionDef') - def visit_Module_FunctionDef_Call(self, module: Module, function: Function, our_locals: OurLocals, exp_type: OurType, node: ast.Call) -> FunctionCall: + def visit_Module_FunctionDef_Call(self, module: Module, function: Function, our_locals: OurLocals, exp_type: OurType, node: ast.Call) -> Union[FunctionCall, UnaryOp]: if node.keywords: _raise_static_error(node, 'Keyword calling not supported') # Yet? @@ -848,6 +846,18 @@ class OurVisitor: struct_constructor = StructConstructor(struct) func = module.functions[struct_constructor.name] + elif node.func.id in WEBASSEMBLY_BUILDIN_FLOAT_OPS: + if not isinstance(exp_type, (OurTypeFloat32, OurTypeFloat64, )): + _raise_static_error(node, f'Cannot make square root result in {exp_type}') + + if 1 != len(node.args): + _raise_static_error(node, f'Function {node.func.id} requires 1 arguments but {len(node.args)} are given') + + return UnaryOp( + exp_type, + 'sqrt', + self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.args[0]), + ) else: if node.func.id not in module.functions: _raise_static_error(node, 'Call to undefined function') From ac0c49a92c039b29ae5ae607976750501a2e6a06 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sun, 19 Jun 2022 16:54:14 +0200 Subject: [PATCH 33/73] Now runs on new code --- TODO.md | 6 ++ py2wasm/compiler.py | 50 +++++++++++++- py2wasm/ourlang.py | 109 ++++++++++++++++++++++++------- tests/integration/helpers.py | 41 ++++++++---- tests/integration/test_simple.py | 25 +++++-- 5 files changed, 189 insertions(+), 42 deletions(-) create mode 100644 TODO.md diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..afe6948 --- /dev/null +++ b/TODO.md @@ -0,0 +1,6 @@ +# TODO + +- Rename ourlang.Struct to OurTypeStruct + - Maybe rename this to a types module? +- Remove references to log_int32_list +- Fix naming in Suite() calls diff --git a/py2wasm/compiler.py b/py2wasm/compiler.py index d04bb2c..7563c75 100644 --- a/py2wasm/compiler.py +++ b/py2wasm/compiler.py @@ -21,16 +21,19 @@ def type_(inp: ourlang.OurType) -> wasm.OurType: if isinstance(inp, ourlang.OurTypeFloat64): return wasm.OurTypeFloat64() - if isinstance(inp, ourlang.Struct): - # Structs are passed as pointer + if isinstance(inp, (ourlang.Struct, ourlang.OurTypeTuple, )): + # Structs and tuples are passed as pointer # And pointers are i32 return wasm.OurTypeInt32() raise NotImplementedError(type_, inp) +# Operators that work for i32, i64, f32, f64 OPERATOR_MAP = { '+': 'add', '-': 'sub', + '*': 'mul', + '==': 'eq', } I32_OPERATOR_MAP = { # TODO: Introduce UInt32 type @@ -121,6 +124,19 @@ def expression(inp: ourlang.Expression) -> Statements: yield wasm.Statement(inp.type.render() + '.load', 'offset=' + str(inp.member.offset)) return + if isinstance(inp, ourlang.AccessTupleMember): + # FIXME: Properly implement this + # inp.type.render() is also a hack that doesn't really work consistently + if not isinstance(inp.type, ( + ourlang.OurTypeInt32, ourlang.OurTypeFloat32, + ourlang.OurTypeInt64, ourlang.OurTypeFloat64, + )): + raise NotImplementedError(inp, inp.type) + + yield from expression(inp.varref) + yield wasm.Statement(inp.type.render() + '.load', 'offset=' + str(inp.member.offset)) + return + raise NotImplementedError(expression, inp) def statement_return(inp: ourlang.StatementReturn) -> Statements: @@ -157,7 +173,14 @@ def function_argument(inp: Tuple[str, ourlang.OurType]) -> wasm.Param: return (inp[0], type_(inp[1]), ) def function(inp: ourlang.Function) -> wasm.Function: - if isinstance(inp, ourlang.StructConstructor): + if isinstance(inp, ourlang.TupleConstructor): + statements = [ + *_generate_tuple_constructor(inp) + ] + locals_ = [ + ('___new_reference___addr', wasm.OurTypeInt32(), ), + ] + elif isinstance(inp, ourlang.StructConstructor): statements = [ *_generate_struct_constructor(inp) ] @@ -219,6 +242,27 @@ def _generate_allocator(mod: ourlang.Module) -> wasm.Function: ], ) +def _generate_tuple_constructor(inp: ourlang.TupleConstructor) -> Statements: + yield wasm.Statement('i32.const', str(inp.tuple.alloc_size())) + yield wasm.Statement('call', '$___new_reference___') + + yield wasm.Statement('local.set', '$___new_reference___addr') + + for member in inp.tuple.members: + # FIXME: Properly implement this + # inp.type.render() is also a hack that doesn't really work consistently + if not isinstance(member.type, ( + ourlang.OurTypeInt32, ourlang.OurTypeFloat32, + ourlang.OurTypeInt64, ourlang.OurTypeFloat64, + )): + raise NotImplementedError + + yield wasm.Statement('local.get', '$___new_reference___addr') + yield wasm.Statement('local.get', f'$arg{member.idx}') + yield wasm.Statement(f'{member.type.render()}.store', 'offset=' + str(member.offset)) + + yield wasm.Statement('local.get', '$___new_reference___addr') + def _generate_struct_constructor(inp: ourlang.StructConstructor) -> Statements: yield wasm.Statement('i32.const', str(inp.struct.alloc_size())) yield wasm.Statement('call', '$___new_reference___') diff --git a/py2wasm/ourlang.py b/py2wasm/ourlang.py index 1855e21..b7c5d68 100644 --- a/py2wasm/ourlang.py +++ b/py2wasm/ourlang.py @@ -59,6 +59,9 @@ class OurTypeInt64(OurType): def render(self) -> str: return 'i64' + def alloc_size(self) -> int: + return 8 + class OurTypeFloat32(OurType): """ The Float type, 32 bits wide @@ -68,6 +71,9 @@ class OurTypeFloat32(OurType): def render(self) -> str: return 'f32' + def alloc_size(self) -> int: + return 4 + class OurTypeFloat64(OurType): """ The Float type, 64 bits wide @@ -77,21 +83,44 @@ class OurTypeFloat64(OurType): def render(self) -> str: return 'f64' + def alloc_size(self) -> int: + return 8 + +class TupleMember: + """ + Represents a tuple member + """ + def __init__(self, idx: int, type_: OurType, offset: int) -> None: + self.idx = idx + self.type = type_ + self.offset = offset + class OurTypeTuple(OurType): """ The tuple type, 64 bits wide """ __slots__ = ('members', ) - members: List[OurType] + members: List[TupleMember] def __init__(self) -> None: self.members = [] def render(self) -> str: - mems = ', '.join(x.render() for x in self.members) + mems = ', '.join(x.type.render() for x in self.members) return f'({mems}, )' + def render_internal_name(self) -> str: + mems = '@'.join(x.type.render() for x in self.members) + assert ' ' not in mems, 'Not implement yet: subtuples' + return f'tuple@{mems}' + + def alloc_size(self) -> int: + return sum( + x.type.alloc_size() + for x in self.members + ) + class Expression: """ An expression within a statement @@ -257,6 +286,9 @@ class FunctionCall(Expression): if isinstance(self.function, StructConstructor): return f'{self.function.struct.name}({args})' + if isinstance(self.function, TupleConstructor): + return f'({args}, )' + return f'{self.function.name}({args})' class AccessStructMember(Expression): @@ -281,21 +313,19 @@ class AccessTupleMember(Expression): """ Access a tuple member for reading of writing """ - __slots__ = ('varref', 'member_idx', 'member_type', ) + __slots__ = ('varref', 'member', ) varref: VariableReference - member_idx: int - member_type: 'OurType' + member: TupleMember - def __init__(self, varref: VariableReference, member_idx: int, member_type: 'OurType') -> None: - super().__init__(member_type) + def __init__(self, varref: VariableReference, member: TupleMember, ) -> None: + super().__init__(member.type) self.varref = varref - self.member_idx = member_idx - self.member_type = member_type + self.member = member def render(self) -> str: - return f'{self.varref.render()}[{self.member_idx}]' + return f'{self.varref.render()}[{self.member.idx}]' class TupleCreation(Expression): """ @@ -450,6 +480,26 @@ class StructConstructor(Function): self.struct = struct +class TupleConstructor(Function): + """ + The constructor method for a tuple + """ + __slots__ = ('tuple', ) + + tuple: OurTypeTuple + + def __init__(self, tuple_: OurTypeTuple) -> None: + name = tuple_.render_internal_name() + + super().__init__(f'@{name}@__init___@', -1) + + self.returns = tuple_ + + for mem in tuple_.members: + self.posonlyargs.append((f'arg{mem.idx}', mem.type, )) + + self.tuple = tuple_ + class StructMember: """ Represents a struct member @@ -823,10 +873,14 @@ class OurVisitor: if len(exp_type.members) != len(node.elts): _raise_static_error(node, f'Expression is expecting a tuple of size {len(exp_type.members)}, but {len(node.elts)} are given') - result = TupleCreation(exp_type) - result.members = [ - self.visit_Module_FunctionDef_expr(module, function, our_locals, arg_type, arg_node) - for arg_node, arg_type in zip(node.elts, exp_type.members) + tuple_constructor = TupleConstructor(exp_type) + + func = module.functions[tuple_constructor.name] + + result = FunctionCall(func) + result.arguments = [ + self.visit_Module_FunctionDef_expr(module, function, our_locals, mem.type, arg_node) + for arg_node, mem in zip(node.elts, exp_type.members) ] return result @@ -930,12 +984,11 @@ class OurVisitor: _raise_static_error(node, f'Index {node.slice.value.value} out of bounds for tuple {node.value.id}') member = node_typ.members[node.slice.value.value] - if exp_type != member: - _raise_static_error(node, f'Expected {exp_type.render()}, got {member.render()} instead') + if exp_type != member.type: + _raise_static_error(node, f'Expected {exp_type.render()}, got {member.type.render()} instead') return AccessTupleMember( VariableReference(node_typ, node.value.id), - node.slice.value.value, member, ) @@ -997,11 +1050,23 @@ class OurVisitor: _raise_static_error(node, 'Must be load context') result = OurTypeTuple() - result.members = [ - self.visit_type(module, elt) - for elt in node.elts - ] - return result + + offset = 0 + + for idx, elt in enumerate(node.elts): + member = TupleMember(idx, self.visit_type(module, elt), offset) + + result.members.append(member) + offset += member.type.alloc_size() + + key = result.render_internal_name() + + if key not in module.types: + module.types[key] = result + constructor = TupleConstructor(result) + module.functions[constructor.name] = constructor + + return module.types[key] raise NotImplementedError(f'{node} as type') diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 634612f..356c8d3 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -15,6 +15,7 @@ import wasmer_compiler_cranelift import wasmtime from py2wasm.utils import our_process, process +from py2wasm.compiler import module DASHES = '-' * 16 @@ -69,23 +70,30 @@ class Suite: Returned is an object with the results set """ + # sys.stderr.write(f'{DASHES} Old Assembly {DASHES}\n') + # + # code_wat_old = process(self.code_py, self.test_name) + # _write_numbered_lines(code_wat_old) + our_module = our_process(self.code_py, self.test_name) + + # Check if code formatting works assert self.code_py == '\n' + our_module.render() # \n for formatting in tests - code_wat = process(self.code_py, self.test_name) + # Compile + wat_module = module(our_module) + + # Render as text + code_wat = wat_module.generate() sys.stderr.write(f'{DASHES} Assembly {DASHES}\n') - line_list = code_wat.split('\n') - line_no_width = len(str(len(line_list))) - for line_no, line_txt in enumerate(line_list): - sys.stderr.write('{} {}\n'.format( - str(line_no + 1).zfill(line_no_width), - line_txt, - )) + _write_numbered_lines(code_wat) + # Compile to assembly code code_wasm = wat2wasm(code_wat) + # Run assembly code return _run_pywasm3(code_wasm, args) def _run_pywasm(code_wasm, args): @@ -117,13 +125,13 @@ def _run_pywasm3(code_wasm, args): rtime = env.new_runtime(1024 * 1024) rtime.load(mod) - sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') - _dump_memory(rtime.get_memory(0).tobytes()) + # sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') + # _dump_memory(rtime.get_memory(0).tobytes()) result.returned_value = rtime.find_function('testEntry')(*args) - sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n') - _dump_memory(rtime.get_memory(0).tobytes()) + # sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n') + # _dump_memory(rtime.get_memory(0).tobytes()) return result @@ -190,3 +198,12 @@ def _dump_memory(mem): sys.stderr.write(f'{idx:08x} {line}\n') prev_line = line + +def _write_numbered_lines(text: str) -> None: + line_list = text.split('\n') + line_no_width = len(str(len(line_list))) + for line_no, line_txt in enumerate(line_list): + sys.stderr.write('{} {}\n'.format( + str(line_no + 1).zfill(line_no_width), + line_txt, + )) diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index 97cd31b..ea281e7 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -51,6 +51,20 @@ def testEntry() -> {type_}: assert 7 == result.returned_value assert TYPE_MAP[type_] == type(result.returned_value) +@pytest.mark.integration_test +@pytest.mark.parametrize('type_', ['f32', 'f64']) +def test_buildins_sqrt(type_): + code_py = f""" +@exported +def testEntry() -> {type_}: + return sqrt(25) +""" + + result = Suite(code_py, 'test_addition').run_code() + + assert 5 == result.returned_value + assert TYPE_MAP[type_] == type(result.returned_value) + @pytest.mark.integration_test @pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) def test_arg(type_): @@ -320,20 +334,21 @@ def helper(shape1: Rectangle, shape2: Rectangle) -> i32: assert [] == result.log_int32_list @pytest.mark.integration_test -def test_tuple_int(): - code_py = """ +@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) +def test_tuple_simple(type_): + code_py = f""" @exported -def testEntry() -> i32: +def testEntry() -> {type_}: return helper((24, 57, 80, )) -def helper(vector: (i32, i32, i32, )) -> i32: +def helper(vector: ({type_}, {type_}, {type_}, )) -> {type_}: return vector[0] + vector[1] + vector[2] """ result = Suite(code_py, 'test_call').run_code() assert 161 == result.returned_value - assert [] == result.log_int32_list + assert TYPE_MAP[type_] == type(result.returned_value) @pytest.mark.integration_test def test_tuple_float(): From 0da309a28047605041154a9683c256d0a0105f88 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sun, 19 Jun 2022 17:04:20 +0200 Subject: [PATCH 34/73] Remove old code --- Makefile | 10 +- examples/.gitignore | 2 + examples/fib.py | 18 + py2wasm/__main__.py | 7 +- py2wasm/ourlang.py | 18 - py2wasm/python.py | 792 ----------------------------------- py2wasm/utils.py | 12 - tests/integration/helpers.py | 7 +- 8 files changed, 32 insertions(+), 834 deletions(-) create mode 100644 examples/.gitignore create mode 100644 examples/fib.py delete mode 100644 py2wasm/python.py diff --git a/Makefile b/Makefile index 5749492..260fe35 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,8 @@ WABT_DIR := /home/johan/Sources/github.com/WebAssembly/wabt WAT2WASM := $(WABT_DIR)/bin/wat2wasm WASM2C := $(WABT_DIR)/bin/wasm2c -%.wat: %.py py2wasm/*.py - python3.8 -m py2wasm $< $@ +%.wat: %.py py2wasm/*.py venv/.done + venv/bin/python -m py2wasm $< $@ %.wasm: %.wat $(WAT2WASM) $^ -o $@ @@ -15,8 +15,8 @@ WASM2C := $(WABT_DIR)/bin/wasm2c # %.exe: %.c # cc $^ -o $@ -I $(WABT_DIR)/wasm2c -server: - python3.8 -m http.server +server: venv/.done + venv/bin/python -m http.server test: venv/.done WAT2WASM=$(WAT2WASM) venv/bin/pytest tests $(TEST_FLAGS) @@ -32,3 +32,5 @@ venv/.done: requirements.txt venv/bin/python3 -m pip install wheel pip --upgrade venv/bin/python3 -m pip install -r $^ touch $@ + +.SECONDARY: # Keep intermediate files diff --git a/examples/.gitignore b/examples/.gitignore new file mode 100644 index 0000000..50765be --- /dev/null +++ b/examples/.gitignore @@ -0,0 +1,2 @@ +*.wasm +*.wat diff --git a/examples/fib.py b/examples/fib.py new file mode 100644 index 0000000..ae15091 --- /dev/null +++ b/examples/fib.py @@ -0,0 +1,18 @@ +def helper(n: i32, a: i32, b: i32) -> i32: + if n < 1: + return a + b + + return helper(n - 1, a + b, a) + +def fib(n: i32) -> i32: + if n == 0: + return 0 + + if n == 1: + return 1 + + return helper(n - 1, 0, 1) + +@exported +def testEntry() -> i32: + return fib(40) diff --git a/py2wasm/__main__.py b/py2wasm/__main__.py index 59757dd..f66b313 100644 --- a/py2wasm/__main__.py +++ b/py2wasm/__main__.py @@ -4,7 +4,8 @@ Functions for using this module from CLI import sys -from .utils import process +from .utils import our_process +from .compiler import module def main(source: str, sink: str) -> int: """ @@ -14,7 +15,9 @@ def main(source: str, sink: str) -> int: with open(source, 'r') as fil: code_py = fil.read() - code_wat = process(code_py, source) + our_module = our_process(code_py, source) + wat_module = module(our_module) + code_wat = wat_module.generate() with open(sink, 'w') as fil: fil.write(code_wat) diff --git a/py2wasm/ourlang.py b/py2wasm/ourlang.py index b7c5d68..622a321 100644 --- a/py2wasm/ourlang.py +++ b/py2wasm/ourlang.py @@ -327,24 +327,6 @@ class AccessTupleMember(Expression): def render(self) -> str: return f'{self.varref.render()}[{self.member.idx}]' -class TupleCreation(Expression): - """ - Create a tuple instance - """ - __slots__ = ('members', ) - - type: OurTypeTuple - members: List[Expression] - - def __init__(self, type_: OurTypeTuple) -> None: - super().__init__(type_) - - self.members = [] - - def render(self) -> str: - mems = ', '. join(x.render() for x in self.members) - return f'({mems}, )' - class Statement: """ A statement within a function diff --git a/py2wasm/python.py b/py2wasm/python.py deleted file mode 100644 index 18d6869..0000000 --- a/py2wasm/python.py +++ /dev/null @@ -1,792 +0,0 @@ -""" -Code for parsing the source (python-alike) code -""" - -from typing import Dict, Generator, List, Optional, Union, Tuple - -import ast - -from . import wasm - -# pylint: disable=C0103,R0201 - -StatementGenerator = Generator[wasm.Statement, None, None] - -WLocals = Dict[str, wasm.OurType] - -class Visitor: - """ - Class to visit a Python syntax tree - - Since we need to visit the whole tree, there's no point in subclassing - the buildin visitor - """ - - def __init__(self) -> None: - self._type_map: WLocals = { - 'None': wasm.OurTypeNone(), - 'bool': wasm.OurTypeBool(), - 'i32': wasm.OurTypeInt32(), - 'i64': wasm.OurTypeInt64(), - 'f32': wasm.OurTypeFloat32(), - 'f64': wasm.OurTypeFloat64(), - - 'i32x4': wasm.OurTypeVectorInt32x4(), - } - - def _get_type(self, name: Union[None, str, ast.expr]) -> wasm.OurType: - if name is None: - name = 'None' - - if isinstance(name, ast.expr): - if isinstance(name, ast.Name): - name = name.id - elif isinstance(name, ast.Tuple): - assert isinstance(name.ctx, ast.Load) - - args: List[wasm.TupleMember] = [] - offset = 0 - for name_arg in name.elts: - arg = wasm.TupleMember( - self._get_type(name_arg), - offset - ) - - args.append(arg) - offset += arg.type.alloc_size() - - return wasm.OurTypeTuple(args) - else: - raise NotImplementedError(f'_get_type(ast.{type(name).__name__})') - - return self._type_map[name] - - def visit_Module(self, node: ast.Module) -> wasm.Module: - """ - Visits a Python module, which results in a wasm Module - """ - assert not node.type_ignores - - module = wasm.Module() - - module.functions.append(wasm.Function( - '___new_reference___', - False, - [ - ('alloc_size', self._get_type('i32')), - ], - [ - ('result', self._get_type('i32')), - ], - self._get_type('i32'), - [ - wasm.Statement('i32.const', '0'), - wasm.Statement('i32.const', '0'), - wasm.Statement('i32.load'), - wasm.Statement('local.tee', '$result', comment='Address for this call'), - wasm.Statement('local.get', '$alloc_size'), - wasm.Statement('i32.add'), - wasm.Statement('i32.store', comment='Address for the next call'), - wasm.Statement('local.get', '$result'), - ] - )) - - # We create functions for these special instructions - # We'll let a future inline optimizer clean this up for us - # TODO: Either a) make a sqrt32 and a sqrt64; - # or b) decide to make the whole language auto-cast - module.functions.append(wasm.Function( - 'sqrt', - False, - [ - ('z', self._type_map['f32']), - ], - [], - self._type_map['f32'], - [ - wasm.Statement('local.get', '$z'), - wasm.Statement('f32.sqrt'), - ] - )) - - # Do a check first for all function definitions - # to get their types. Otherwise you cannot call - # a method that you haven't defined just yet, - # even if it is in the same file - function_body_map: Dict[ast.FunctionDef, wasm.Function] = {} - for stmt in node.body: - if isinstance(stmt, ast.FunctionDef): - wnode = self.pre_visit_FunctionDef(module, stmt) - - if isinstance(wnode, wasm.Import): - module.imports.append(wnode) - else: - module.functions.append(wnode) - function_body_map[stmt] = wnode - continue - - if isinstance(stmt, ast.ClassDef): - wclass = self.pre_visit_ClassDef(module, stmt) - self._type_map[wclass.name] = wclass - continue - - # No other pre visits to do - - for stmt in node.body: - if isinstance(stmt, ast.FunctionDef): - if stmt in function_body_map: - self.parse_FunctionDef_body(module, function_body_map[stmt], stmt) - # else: It's an import, no actual body to parse - continue - - if isinstance(stmt, ast.ClassDef): - continue - - raise NotImplementedError(stmt) - - return module - - def pre_visit_FunctionDef( - self, - module: wasm.Module, - node: ast.FunctionDef, - ) -> Union[wasm.Import, wasm.Function]: - """ - A Python function definition with the @external decorator is returned as import. - - Other functions are returned as (exported) functions. - - Nested / dynamicly created functions are not yet supported - """ - exported = False - - if node.decorator_list: - assert 1 == len(node.decorator_list) - - decorator = node.decorator_list[0] - if isinstance(decorator, ast.Name): - assert 'exported' == decorator.id - exported = True - elif isinstance(decorator, ast.Call): - call = decorator - - assert not node.type_comment - assert 1 == len(node.body) - assert isinstance(node.body[0], ast.Expr) - assert isinstance(node.body[0].value, ast.Ellipsis) - - assert isinstance(call.func, ast.Name) - assert 'imported' == call.func.id - - assert not call.keywords - - mod, name, intname = self._parse_import_decorator(node.name, call.args) - - return wasm.Import(mod, name, intname, self._parse_import_args(node.args)) - else: - raise NotImplementedError - - result = self._get_type(node.returns) - - assert not node.args.vararg - assert not node.args.kwonlyargs - assert not node.args.kw_defaults - assert not node.args.kwarg - assert not node.args.defaults - - params: List[wasm.Param] = [] - for arg in [*node.args.posonlyargs, *node.args.args]: - params.append((arg.arg, self._get_type(arg.annotation), )) - - locals_: List[wasm.Param] = [ - ('___new_reference___addr', self._get_type('i32')), # For the ___new_reference___ method - ] - - return wasm.Function(node.name, exported, params, locals_, result, []) - - def parse_FunctionDef_body( - self, - module: wasm.Module, - func: wasm.Function, - node: ast.FunctionDef, - ) -> None: - """ - Parses the function body - """ - wlocals: WLocals = dict(func.params) - - statements: List[wasm.Statement] = [] - for py_stmt in node.body: - statements.extend( - self.visit_stmt(module, node, wlocals, py_stmt) - ) - - func.statements = statements - - def pre_visit_ClassDef( - self, - module: wasm.Module, - node: ast.ClassDef, - ) -> wasm.OurTypeClass: - """ - TODO: Document this - """ - del module - - if node.bases or node.keywords or node.decorator_list: - raise NotImplementedError - - members: List[wasm.ClassMember] = [] - - offset = 0 - for stmt in node.body: - if not isinstance(stmt, ast.AnnAssign): - raise NotImplementedError - - if not isinstance(stmt.target, ast.Name): - raise NotImplementedError - - if not isinstance(stmt.annotation, ast.Name): - raise NotImplementedError('Cannot recurse classes yet') - - if stmt.annotation.id != 'i32': - raise NotImplementedError - - if stmt.value is None: - default = None - else: - if not isinstance(stmt.value, ast.Constant): - raise NotImplementedError - - if not isinstance(stmt.value.value, int): - raise NotImplementedError - - default = wasm.Constant(stmt.value.value) - - member = wasm.ClassMember( - stmt.target.id, self._get_type(stmt.annotation), offset, default - ) - - members.append(member) - offset += member.type.alloc_size() - - return wasm.OurTypeClass(node.name, members) - - def visit_stmt( - self, - module: wasm.Module, - func: ast.FunctionDef, - wlocals: WLocals, - stmt: ast.stmt, - ) -> StatementGenerator: - """ - Visits a statement node within a function - """ - if isinstance(stmt, ast.Return): - return self.visit_Return(module, func, wlocals, stmt) - - if isinstance(stmt, ast.Expr): - assert isinstance(stmt.value, ast.Call) - - return self.visit_Call(module, wlocals, self._get_type('None'), stmt.value) - - if isinstance(stmt, ast.If): - return self.visit_If(module, func, wlocals, stmt) - - raise NotImplementedError(stmt) - - def visit_Return( - self, - module: wasm.Module, - func: ast.FunctionDef, - wlocals: WLocals, - stmt: ast.Return, - ) -> StatementGenerator: - """ - Visits a Return node - """ - - assert stmt.value is not None - - return_type = self._get_type(func.returns) - - yield from self.visit_expr(module, wlocals, return_type, stmt.value) - yield wasm.Statement('return') - - def visit_If( - self, - module: wasm.Module, - func: ast.FunctionDef, - wlocals: WLocals, - stmt: ast.If, - ) -> StatementGenerator: - """ - Visits an If node - """ - yield from self.visit_expr( - module, - wlocals, - self._get_type('bool'), - stmt.test, - ) - - yield wasm.Statement('if') - - for py_stmt in stmt.body: - yield from self.visit_stmt(module, func, wlocals, py_stmt) - - yield wasm.Statement('else') - - for py_stmt in stmt.orelse: - yield from self.visit_stmt(module, func, wlocals, py_stmt) - - yield wasm.Statement('end') - - def visit_expr( - self, - module: wasm.Module, - wlocals: WLocals, - exp_type: wasm.OurType, - node: ast.expr, - ) -> StatementGenerator: - """ - Visit an expression node - """ - if isinstance(node, ast.BinOp): - return self.visit_BinOp(module, wlocals, exp_type, node) - - if isinstance(node, ast.UnaryOp): - return self.visit_UnaryOp(module, wlocals, exp_type, node) - - if isinstance(node, ast.Compare): - return self.visit_Compare(module, wlocals, exp_type, node) - - if isinstance(node, ast.Call): - return self.visit_Call(module, wlocals, exp_type, node) - - if isinstance(node, ast.Constant): - return self.visit_Constant(exp_type, node) - - if isinstance(node, ast.Attribute): - return self.visit_Attribute(module, wlocals, exp_type, node) - - if isinstance(node, ast.Subscript): - return self.visit_Subscript(module, wlocals, exp_type, node) - - if isinstance(node, ast.Name): - return self.visit_Name(wlocals, exp_type, node) - - if isinstance(node, ast.Tuple): - return self.visit_Tuple(module, wlocals, exp_type, node) - - raise NotImplementedError(node) - - def visit_UnaryOp( - self, - module: wasm.Module, - wlocals: WLocals, - exp_type: wasm.OurType, - node: ast.UnaryOp, - ) -> StatementGenerator: - """ - Visits a UnaryOp node as (part of) an expression - """ - if isinstance(node.op, ast.UAdd): - return self.visit_expr(module, wlocals, exp_type, node.operand) - - if isinstance(node.op, ast.USub): - if not isinstance(node.operand, ast.Constant): - raise NotImplementedError(node.operand) - - if not isinstance(node.operand.value, int): - raise NotImplementedError(node.operand.value) - - return self.visit_Constant(exp_type, ast.Constant(-node.operand.value)) - - raise NotImplementedError(node.op) - - def visit_BinOp( - self, - module: wasm.Module, - wlocals: WLocals, - exp_type: wasm.OurType, - node: ast.BinOp, - ) -> StatementGenerator: - """ - Visits a BinOp node as (part of) an expression - """ - yield from self.visit_expr(module, wlocals, exp_type, node.left) - yield from self.visit_expr(module, wlocals, exp_type, node.right) - - if isinstance(node.op, ast.Add): - yield wasm.Statement('{}.add'.format(exp_type.to_wasm())) - return - - if isinstance(node.op, ast.Mult): - yield wasm.Statement('{}.mul'.format(exp_type.to_wasm())) - return - - if isinstance(node.op, ast.Sub): - yield wasm.Statement('{}.sub'.format(exp_type.to_wasm())) - return - - raise NotImplementedError(node.op) - - def visit_Compare( - self, - module: wasm.Module, - wlocals: WLocals, - exp_type: wasm.OurType, - node: ast.Compare, - ) -> StatementGenerator: - """ - Visits a Compare node as (part of) an expression - """ - assert isinstance(exp_type, wasm.OurTypeBool) - - if 1 != len(node.ops) or 1 != len(node.comparators): - raise NotImplementedError - - yield from self.visit_expr(module, wlocals, self._get_type('i32'), node.left) - yield from self.visit_expr(module, wlocals, self._get_type('i32'), node.comparators[0]) - - if isinstance(node.ops[0], ast.Lt): - yield wasm.Statement('i32.lt_s') - return - - if isinstance(node.ops[0], ast.Eq): - yield wasm.Statement('i32.eq') - return - - if isinstance(node.ops[0], ast.Gt): - yield wasm.Statement('i32.gt_s') - return - - raise NotImplementedError(node.ops) - - def visit_Call( - self, - module: wasm.Module, - wlocals: WLocals, - exp_type: wasm.OurType, - node: ast.Call, - ) -> StatementGenerator: - """ - Visits a Call node as (part of) an expression - """ - assert isinstance(node.func, ast.Name) - assert not node.keywords - - called_name = node.func.id - - if called_name in self._type_map: - klass = self._type_map[called_name] - if isinstance(klass, wasm.OurTypeClass): - return self.visit_Call_class(module, wlocals, exp_type, node, klass) - - search_list: List[Union[wasm.Function, wasm.Import]] - search_list = [ - *module.functions, - *module.imports, - ] - - called_func_list = [ - x - for x in search_list - if x.name == called_name - ] - - assert 1 == len(called_func_list), \ - 'Could not find function {}'.format(node.func.id) - - return self.visit_Call_func(module, wlocals, exp_type, node, called_func_list[0]) - - def visit_Call_class( - self, - module: wasm.Module, - wlocals: WLocals, - exp_type: wasm.OurType, - node: ast.Call, - cls: wasm.OurTypeClass, - ) -> StatementGenerator: - """ - Visits a Call node as (part of) an expression - - This instantiates the class - """ - - # TODO: free call - - yield wasm.Statement('i32.const', str(cls.alloc_size())) - yield wasm.Statement('call', '$___new_reference___') - - yield wasm.Statement('local.set', '$___new_reference___addr') - - for member, arg in zip(cls.members, node.args): - if not isinstance(arg, ast.Constant): - raise NotImplementedError - - yield wasm.Statement('local.get', '$___new_reference___addr') - yield wasm.Statement(f'{member.type.to_wasm()}.const', str(arg.value)) - yield wasm.Statement(f'{member.type.to_wasm()}.store', 'offset=' + str(member.offset)) - - yield wasm.Statement('local.get', '$___new_reference___addr') - - def visit_Call_func( - self, - module: wasm.Module, - wlocals: WLocals, - exp_type: wasm.OurType, - node: ast.Call, - func: Union[wasm.Function, wasm.Import], - ) -> StatementGenerator: - """ - Visits a Call node as (part of) an expression - """ - assert isinstance(node.func, ast.Name) - - called_name = node.func.id - called_params = func.params - called_result = func.result - - assert exp_type == called_result, \ - f'Function does not match expected type; {called_name} returns {called_result.to_python()} but {exp_type.to_python()} is expected' - - assert len(called_params) == len(node.args), \ - '{}:{} Function {} requires {} arguments, but {} are supplied'.format( - node.lineno, node.col_offset, - called_name, len(called_params), len(node.args), - ) - - for (_, exp_expr_type), expr in zip(called_params, node.args): - yield from self.visit_expr(module, wlocals, exp_expr_type, expr) - - yield wasm.Statement( - 'call', - '${}'.format(called_name), - ) - - def visit_Constant(self, exp_type: wasm.OurType, node: ast.Constant) -> StatementGenerator: - """ - Visits a Constant node as (part of) an expression - """ - if isinstance(exp_type, wasm.OurTypeInt32): - assert isinstance(node.value, int) - assert -2147483648 <= node.value <= 2147483647 - - yield wasm.Statement('i32.const', str(node.value)) - return - - if isinstance(exp_type, wasm.OurTypeInt64): - assert isinstance(node.value, int) - assert -9223372036854775808 <= node.value <= 9223372036854775807 - - yield wasm.Statement('i64.const', str(node.value)) - return - - if isinstance(exp_type, wasm.OurTypeFloat32): - if isinstance(node.value, int): - node.value = float(node.value) - - assert isinstance(node.value, float), \ - f'Expression expected a float, but received {node.value}' - # TODO: Size check? - - yield wasm.Statement('f32.const', node.value.hex()) - return - - if isinstance(exp_type, wasm.OurTypeFloat64): - if isinstance(node.value, int): - node.value = float(node.value) - - assert isinstance(node.value, float) - # TODO: Size check? - - yield wasm.Statement('f64.const', node.value.hex()) - return - - raise NotImplementedError(exp_type) - - def visit_Attribute( - self, - module: wasm.Module, - wlocals: WLocals, - exp_type: wasm.OurType, - node: ast.Attribute, - ) -> StatementGenerator: - """ - Visits an Attribute node as (part of) an expression - """ - if not isinstance(node.value, ast.Name): - raise NotImplementedError - - if not isinstance(node.ctx, ast.Load): - raise NotImplementedError - - cls = wlocals[node.value.id] - - assert isinstance(cls, wasm.OurTypeClass), f'Cannot take property of {cls}' - - member_list = [ - x - for x in cls.members - if x.name == node.attr - ] - - assert len(member_list) == 1 - member = member_list[0] - - yield wasm.Statement('local.get', '$' + node.value.id) - yield wasm.Statement(exp_type.to_wasm() + '.load', 'offset=' + str(member.offset)) - - def visit_Subscript( - self, - module: wasm.Module, - wlocals: WLocals, - exp_type: wasm.OurType, - node: ast.Subscript, - ) -> StatementGenerator: - """ - Visits an Subscript node as (part of) an expression - """ - if not isinstance(node.value, ast.Name): - raise NotImplementedError - - if not isinstance(node.slice, ast.Index): - raise NotImplementedError - - if not isinstance(node.slice.value, ast.Constant): - raise NotImplementedError - - if not isinstance(node.slice.value.value, int): - raise NotImplementedError - - if not isinstance(node.ctx, ast.Load): - raise NotImplementedError - - tpl_idx = node.slice.value.value - - tpl = wlocals[node.value.id] - - assert isinstance(tpl, wasm.OurTypeTuple), f'Cannot take index of {tpl}' - - assert 0 <= tpl_idx < len(tpl.members), \ - f'Tuple index out of bounds; {node.value.id} has {len(tpl.members)} members' - - member = tpl.members[tpl_idx] - - yield wasm.Statement('local.get', '$' + node.value.id) - yield wasm.Statement(exp_type.to_wasm() + '.load', 'offset=' + str(member.offset)) - - def visit_Tuple( - self, - module: wasm.Module, - wlocals: WLocals, - exp_type: wasm.OurType, - node: ast.Tuple, - ) -> StatementGenerator: - """ - Visits an Tuple node as (part of) an expression - """ - if isinstance(exp_type, wasm.OurTypeVector): - assert isinstance(exp_type, wasm.OurTypeVectorInt32x4), 'Not implemented yet' - assert len(node.elts) == 4, 'Not implemented yet' - - values: List[str] = [] - for arg in node.elts: - assert isinstance(arg, ast.Constant) - assert isinstance(arg.value, int) - values.append(str(arg.value)) - - yield wasm.Statement('v128.const', 'i32x4', *values) - return - - assert isinstance(exp_type, wasm.OurTypeTuple), \ - f'Expression is expecting {exp_type}, not a tuple' - - # TODO: free call - - tpl = exp_type - - yield wasm.Statement('nop', comment='Start tuple allocation') - - yield wasm.Statement('i32.const', str(tpl.alloc_size())) - yield wasm.Statement('call', '$___new_reference___') - yield wasm.Statement('local.set', '$___new_reference___addr', comment='Allocate for tuple') - - for member, arg in zip(tpl.members, node.elts): - yield wasm.Statement('local.get', '$___new_reference___addr') - - yield from self.visit_expr(module, wlocals, member.type, arg) - - yield wasm.Statement(f'{member.type.to_wasm()}.store', 'offset=' + str(member.offset), - comment='Write tuple value to memory') - - yield wasm.Statement('local.get', '$___new_reference___addr', comment='Store tuple address on stack') - - def visit_Name(self, wlocals: WLocals, exp_type: wasm.OurType, node: ast.Name) -> StatementGenerator: - """ - Visits a Name node as (part of) an expression - """ - assert node.id in wlocals - - if (isinstance(exp_type, wasm.OurTypeInt64) - and isinstance(wlocals[node.id], wasm.OurTypeInt32)): - yield wasm.Statement('local.get', '${}'.format(node.id)) - yield wasm.Statement('i64.extend_i32_s') - return - - if (isinstance(exp_type, wasm.OurTypeFloat64) - and isinstance(wlocals[node.id], wasm.OurTypeFloat32)): - yield wasm.Statement('local.get', '${}'.format(node.id)) - yield wasm.Statement('f64.promote_f32') - return - - assert exp_type == wlocals[node.id], (exp_type, wlocals[node.id], ) - yield wasm.Statement( - 'local.get', - '${}'.format(node.id), - ) - - def _parse_import_decorator(self, func_name: str, args: List[ast.expr]) -> Tuple[str, str, str]: - """ - Parses an @import decorator - """ - - assert 0 < len(args) < 3 - str_args = [ - arg.value - for arg in args - if isinstance(arg, ast.Constant) - and isinstance(arg.value, str) - ] - assert len(str_args) == len(args) - - module = str_args.pop(0) - if str_args: - name = str_args.pop(0) - else: - name = func_name - - return module, name, func_name - - def _parse_import_args(self, args: ast.arguments) -> List[wasm.Param]: - """ - Parses the arguments for an @imported method - """ - - assert not args.vararg - assert not args.kwonlyargs - assert not args.kw_defaults - assert not args.kwarg - assert not args.defaults # Maybe support this later on - - arg_list = [ - *args.posonlyargs, - *args.args, - ] - - return [ - (arg.arg, self._get_type(arg.annotation)) - for arg in arg_list - ] diff --git a/py2wasm/utils.py b/py2wasm/utils.py index e93d0b5..cc9e0a4 100644 --- a/py2wasm/utils.py +++ b/py2wasm/utils.py @@ -5,18 +5,6 @@ Utility functions import ast from .ourlang import OurVisitor, Module -from .python import Visitor - -def process(source: str, input_name: str) -> str: - """ - Processes the python code into web assembly code - """ - res = ast.parse(source, input_name) - - visitor = Visitor() - module = visitor.visit_Module(res) - - return module.generate() def our_process(source: str, input_name: str) -> Module: """ diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 356c8d3..d9f299e 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -14,7 +14,7 @@ import wasmer_compiler_cranelift import wasmtime -from py2wasm.utils import our_process, process +from py2wasm.utils import our_process from py2wasm.compiler import module DASHES = '-' * 16 @@ -70,11 +70,6 @@ class Suite: Returned is an object with the results set """ - # sys.stderr.write(f'{DASHES} Old Assembly {DASHES}\n') - # - # code_wat_old = process(self.code_py, self.test_name) - # _write_numbered_lines(code_wat_old) - our_module = our_process(self.code_py, self.test_name) # Check if code formatting works From b28df7fa743be4c2a85cdb6855194dabdd429f38 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Fri, 24 Jun 2022 18:52:43 +0200 Subject: [PATCH 35/73] Fix: Could not both export and use function Added HTML pages for fib example --- Makefile | 6 ++++-- examples/fib.html | 34 ++++++++++++++++++++++++++++++++++ examples/fib.py | 7 ++++--- examples/index.html | 11 +++++++++++ py2wasm/compiler.py | 10 ++++++++++ py2wasm/wasm.py | 7 ++++++- 6 files changed, 69 insertions(+), 6 deletions(-) create mode 100644 examples/fib.html create mode 100644 examples/index.html diff --git a/Makefile b/Makefile index 260fe35..7ccbb41 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,8 @@ WASM2C := $(WABT_DIR)/bin/wasm2c # %.exe: %.c # cc $^ -o $@ -I $(WABT_DIR)/wasm2c -server: venv/.done - venv/bin/python -m http.server +examples: venv/.done $(subst .py,.wasm,$(wildcard examples/*.py)) + venv/bin/python3 -m http.server --directory examples test: venv/.done WAT2WASM=$(WAT2WASM) venv/bin/pytest tests $(TEST_FLAGS) @@ -34,3 +34,5 @@ venv/.done: requirements.txt touch $@ .SECONDARY: # Keep intermediate files + +.PHONY: examples diff --git a/examples/fib.html b/examples/fib.html new file mode 100644 index 0000000..841e8c7 --- /dev/null +++ b/examples/fib.html @@ -0,0 +1,34 @@ + + +Examples - Fibonacci + + +

Fibonacci

+ +Source - WebAssembly + +
+ + + + + + diff --git a/examples/fib.py b/examples/fib.py index ae15091..8536245 100644 --- a/examples/fib.py +++ b/examples/fib.py @@ -1,10 +1,11 @@ -def helper(n: i32, a: i32, b: i32) -> i32: +def helper(n: i64, a: i64, b: i64) -> i64: if n < 1: return a + b return helper(n - 1, a + b, a) -def fib(n: i32) -> i32: +@exported +def fib(n: i64) -> i64: if n == 0: return 0 @@ -14,5 +15,5 @@ def fib(n: i32) -> i32: return helper(n - 1, 0, 1) @exported -def testEntry() -> i32: +def testEntry() -> i64: return fib(40) diff --git a/examples/index.html b/examples/index.html new file mode 100644 index 0000000..0093a41 --- /dev/null +++ b/examples/index.html @@ -0,0 +1,11 @@ + + +Examples + + +

Examples

+ + + diff --git a/py2wasm/compiler.py b/py2wasm/compiler.py index 7563c75..c8fe879 100644 --- a/py2wasm/compiler.py +++ b/py2wasm/compiler.py @@ -43,6 +43,13 @@ I32_OPERATOR_MAP = { # TODO: Introduce UInt32 type '>=': 'ge_s', } +I64_OPERATOR_MAP = { # TODO: Introduce UInt32 type + '<': 'lt_s', + '>': 'gt_s', + '<=': 'le_s', + '>=': 'ge_s', +} + def expression(inp: ourlang.Expression) -> Statements: if isinstance(inp, ourlang.ConstantInt32): yield wasm.Statement('i32.const', str(inp.value)) @@ -79,6 +86,9 @@ def expression(inp: ourlang.Expression) -> Statements: if operator := OPERATOR_MAP.get(inp.operator, None): yield wasm.Statement(f'i64.{operator}') return + if operator := I64_OPERATOR_MAP.get(inp.operator, None): + yield wasm.Statement(f'i64.{operator}') + return if isinstance(inp.type, ourlang.OurTypeFloat32): if operator := OPERATOR_MAP.get(inp.operator, None): yield wasm.Statement(f'f32.{operator}') diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py index 49612cd..75c4567 100644 --- a/py2wasm/wasm.py +++ b/py2wasm/wasm.py @@ -216,7 +216,12 @@ class Function: """ Generates the text version """ - header = ('(export "{}")' if self.exported else '${}').format(self.name) + header = f'${self.name}' # Name for internal use + + if self.exported: + # Name for external use + # TODO: Consider: Make exported('export_name') work + header += f' (export "{self.name}")' for nam, typ in self.params: header += f' (param ${nam} {typ.to_wasm()})' From 0afab89796c2ac28dba8547367eb5495b815e396 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Fri, 24 Jun 2022 19:48:07 +0200 Subject: [PATCH 36/73] Memory access from outside with example setup --- examples/buffer.html | 54 ++++++++++++++++++++++++++++++++++++++++++++ examples/buffer.py | 3 +++ examples/fib.html | 6 ++--- examples/index.html | 5 ++++ py2wasm/compiler.py | 2 +- py2wasm/wasm.py | 2 +- 6 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 examples/buffer.html create mode 100644 examples/buffer.py diff --git a/examples/buffer.html b/examples/buffer.html new file mode 100644 index 0000000..c5e13ba --- /dev/null +++ b/examples/buffer.html @@ -0,0 +1,54 @@ + + +Examples - Buffer + + +

Buffer

+ +List - Source - WebAssembly + +
+ + + + + + diff --git a/examples/buffer.py b/examples/buffer.py new file mode 100644 index 0000000..6700f6c --- /dev/null +++ b/examples/buffer.py @@ -0,0 +1,3 @@ +@exported +def sum() -> i32: # TODO: Should be [i32] -> i32 + return 0 # TODO: Implement me diff --git a/examples/fib.html b/examples/fib.html index 841e8c7..8d62880 100644 --- a/examples/fib.html +++ b/examples/fib.html @@ -5,7 +5,7 @@

Fibonacci

-Source - WebAssembly +List - Source - WebAssembly
@@ -16,13 +16,13 @@ let importObject = {}; let results = document.getElementById('results'); WebAssembly.instantiateStreaming(fetch('fib.wasm'), importObject) - .then(fib => { + .then(app => { // 93: 7540113804746346429 // i64: 9223372036854775807 // 94: 19740274219868223167 for(let i = BigInt(1); i < 93; ++i) { let span = document.createElement('span'); - span.innerHTML = 'fib(' + i + ') = ' + fib.instance.exports.fib(i); + span.innerHTML = 'fib(' + i + ') = ' + app.instance.exports.fib(i); results.appendChild(span); let br = document.createElement('br'); results.appendChild(br); diff --git a/examples/index.html b/examples/index.html index 0093a41..29e73c8 100644 --- a/examples/index.html +++ b/examples/index.html @@ -4,8 +4,13 @@

Examples

+

Standard

+

Technical

+ diff --git a/py2wasm/compiler.py b/py2wasm/compiler.py index c8fe879..40263a2 100644 --- a/py2wasm/compiler.py +++ b/py2wasm/compiler.py @@ -232,7 +232,7 @@ def module(inp: ourlang.Module) -> wasm.Module: def _generate_allocator(mod: ourlang.Module) -> wasm.Function: return wasm.Function( '___new_reference___', - False, + True, [ ('alloc_size', type_(mod.types['i32']), ), ], diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py index 75c4567..049df62 100644 --- a/py2wasm/wasm.py +++ b/py2wasm/wasm.py @@ -242,7 +242,7 @@ class ModuleMemory: self.data = data def generate(self) -> str: - return '(memory 1)\n (data (memory 0) (i32.const 0) "{}")\n'.format( + return '(memory 1)\n (data (memory 0) (i32.const 0) "{}")\n (export "memory" (memory 0))\n'.format( ''.join( f'\\{x:02x}' for x in self.data From 467d409d8000f04d9f6cfd4137b177776c7b492b Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Fri, 24 Jun 2022 21:49:27 +0200 Subject: [PATCH 37/73] Add tests for static checks --- py2wasm/ourlang.py | 29 +++++++----- tests/integration/test_static_checking.py | 56 +++++++++++++++++++++++ 2 files changed, 74 insertions(+), 11 deletions(-) create mode 100644 tests/integration/test_static_checking.py diff --git a/py2wasm/ourlang.py b/py2wasm/ourlang.py index 622a321..ba87aec 100644 --- a/py2wasm/ourlang.py +++ b/py2wasm/ourlang.py @@ -97,7 +97,7 @@ class TupleMember: class OurTypeTuple(OurType): """ - The tuple type, 64 bits wide + The tuple type """ __slots__ = ('members', ) @@ -842,8 +842,14 @@ class OurVisitor: if not isinstance(node.ctx, ast.Load): _raise_static_error(node, 'Must be load context') - if node.id in our_locals: - return VariableReference(our_locals[node.id], node.id) + if node.id not in our_locals: + _raise_static_error(node, 'Undefined variable') + + act_type = our_locals[node.id] + if exp_type != act_type: + _raise_static_error(node, f'Expected {exp_type.render()}, {node.id} is actually {act_type.render()}') + + return VariableReference(act_type, node.id) if isinstance(node, ast.Tuple): if not isinstance(node.ctx, ast.Load): @@ -884,7 +890,7 @@ class OurVisitor: func = module.functions[struct_constructor.name] elif node.func.id in WEBASSEMBLY_BUILDIN_FLOAT_OPS: if not isinstance(exp_type, (OurTypeFloat32, OurTypeFloat64, )): - _raise_static_error(node, f'Cannot make square root result in {exp_type}') + _raise_static_error(node, f'Cannot make {node.func.id} result in {exp_type}') if 1 != len(node.args): _raise_static_error(node, f'Function {node.func.id} requires 1 arguments but {len(node.args)} are given') @@ -901,7 +907,7 @@ class OurVisitor: func = module.functions[node.func.id] if func.returns != exp_type: - _raise_static_error(node, f'Function {node.func.id} does not return {exp_type.render()}') + _raise_static_error(node, f'Expected {exp_type.render()}, {func.name} actually returns {func.returns.render()}') if len(func.posonlyargs) != len(node.args): _raise_static_error(node, f'Function {node.func.id} requires {len(func.posonlyargs)} arguments but {len(node.args)} are given') @@ -932,7 +938,7 @@ class OurVisitor: _raise_static_error(node, f'{node_typ.name} has no attribute {node.attr}') if exp_type != member.type: - _raise_static_error(node, f'Expected {exp_type.render()}, got {member.type.render()} instead') + _raise_static_error(node, f'Expected {exp_type.render()}, {node.value.id}.{member.name} is actually {member.type.render()}') return AccessStructMember( VariableReference(node_typ, node.value.id), @@ -949,7 +955,8 @@ class OurVisitor: if not isinstance(node.slice.value, ast.Constant): _raise_static_error(node, 'Must subscript using a constant index') # FIXME: Implement variable indexes - if not isinstance(node.slice.value.value, int): + idx = node.slice.value.value + if not isinstance(idx, int): _raise_static_error(node, 'Must subscript using a constant integer index') if not isinstance(node.ctx, ast.Load): @@ -962,12 +969,12 @@ class OurVisitor: if not isinstance(node_typ, OurTypeTuple): _raise_static_error(node, f'Cannot take index of non-tuple {node.value.id}') - if len(node_typ.members) <= node.slice.value.value: - _raise_static_error(node, f'Index {node.slice.value.value} out of bounds for tuple {node.value.id}') + if len(node_typ.members) <= idx: + _raise_static_error(node, f'Index {idx} out of bounds for tuple {node.value.id}') - member = node_typ.members[node.slice.value.value] + member = node_typ.members[idx] if exp_type != member.type: - _raise_static_error(node, f'Expected {exp_type.render()}, got {member.type.render()} instead') + _raise_static_error(node, f'Expected {exp_type.render()}, {node.value.id}[{idx}] is actually {member.type.render()}') return AccessTupleMember( VariableReference(node_typ, node.value.id), diff --git a/tests/integration/test_static_checking.py b/tests/integration/test_static_checking.py new file mode 100644 index 0000000..f3de90b --- /dev/null +++ b/tests/integration/test_static_checking.py @@ -0,0 +1,56 @@ +import pytest + +from py2wasm.utils import our_process + +from py2wasm.ourlang import StaticError + +@pytest.mark.integration_test +@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) +def test_type_mismatch_function_argument(type_): + code_py = f""" +def helper(a: {type_}) -> (i32, i32, ): + return a +""" + + with pytest.raises(StaticError, match=f'Static error on line 3: Expected \\(i32, i32, \\), a is actually {type_}'): + our_process(code_py, 'test') + +@pytest.mark.integration_test +@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) +def test_type_mismatch_struct_member(type_): + code_py = f""" +class Struct: + param: {type_} + +def testEntry(arg: Struct) -> (i32, i32, ): + return arg.param +""" + + with pytest.raises(StaticError, match=f'Static error on line 6: Expected \\(i32, i32, \\), arg.param is actually {type_}'): + our_process(code_py, 'test') + +@pytest.mark.integration_test +@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) +def test_type_mismatch_tuple_member(type_): + code_py = f""" +def testEntry(arg: ({type_}, )) -> (i32, i32, ): + return arg[0] +""" + + with pytest.raises(StaticError, match=f'Static error on line 3: Expected \\(i32, i32, \\), arg\\[0\\] is actually {type_}'): + our_process(code_py, 'test') + +@pytest.mark.integration_test +@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) +def test_type_mismatch_function_result(type_): + code_py = f""" +def helper() -> {type_}: + return 1 + +@exported +def testEntry() -> (i32, i32, ): + return helper() +""" + + with pytest.raises(StaticError, match=f'Static error on line 7: Expected \\(i32, i32, \\), helper actually returns {type_}'): + our_process(code_py, 'test') From 374231d206c802136d9241bbf06e2783a13c5820 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 25 Jun 2022 20:45:33 +0200 Subject: [PATCH 38/73] bytes, u8 types --- py2wasm/compiler.py | 106 +++++++++++++++++++---- py2wasm/ourlang.py | 85 +++++++++++++++++- tests/integration/helpers.py | 47 +++++++++- tests/integration/test_runtime_checks.py | 15 ++++ tests/integration/test_simple.py | 56 ++++++++++-- 5 files changed, 276 insertions(+), 33 deletions(-) create mode 100644 tests/integration/test_runtime_checks.py diff --git a/py2wasm/compiler.py b/py2wasm/compiler.py index 40263a2..4b71bae 100644 --- a/py2wasm/compiler.py +++ b/py2wasm/compiler.py @@ -9,6 +9,11 @@ from . import wasm Statements = Generator[wasm.Statement, None, None] def type_(inp: ourlang.OurType) -> wasm.OurType: + if isinstance(inp, ourlang.OurTypeUInt8): + # WebAssembly has only support for 32 and 64 bits + # So we need to store more memory per byte + return wasm.OurTypeInt32() + if isinstance(inp, ourlang.OurTypeInt32): return wasm.OurTypeInt32() @@ -21,7 +26,7 @@ def type_(inp: ourlang.OurType) -> wasm.OurType: if isinstance(inp, ourlang.OurTypeFloat64): return wasm.OurTypeFloat64() - if isinstance(inp, (ourlang.Struct, ourlang.OurTypeTuple, )): + if isinstance(inp, (ourlang.Struct, ourlang.OurTypeTuple, ourlang.OurTypeBytes)): # Structs and tuples are passed as pointer # And pointers are i32 return wasm.OurTypeInt32() @@ -51,6 +56,10 @@ I64_OPERATOR_MAP = { # TODO: Introduce UInt32 type } def expression(inp: ourlang.Expression) -> Statements: + if isinstance(inp, ourlang.ConstantUInt8): + yield wasm.Statement('i32.const', str(inp.value)) + return + if isinstance(inp, ourlang.ConstantInt32): yield wasm.Statement('i32.const', str(inp.value)) return @@ -112,6 +121,12 @@ def expression(inp: ourlang.Expression) -> Statements: yield wasm.Statement(f'f64.{inp.operator}') return + if isinstance(inp.type, ourlang.OurTypeInt32): + if inp.operator == 'len': + if isinstance(inp.right.type, ourlang.OurTypeBytes): + yield wasm.Statement('i32.load') + return + raise NotImplementedError(expression, inp.type, inp.operator) if isinstance(inp, ourlang.FunctionCall): @@ -121,17 +136,33 @@ def expression(inp: ourlang.Expression) -> Statements: yield wasm.Statement('call', '${}'.format(inp.function.name)) return - if isinstance(inp, ourlang.AccessStructMember): - # FIXME: Properly implement this - # inp.type.render() is also a hack that doesn't really work consistently - if not isinstance(inp.type, ( - ourlang.OurTypeInt32, ourlang.OurTypeFloat32, - ourlang.OurTypeInt64, ourlang.OurTypeFloat64, - )): + if isinstance(inp, ourlang.AccessBytesIndex): + if not isinstance(inp.type, ourlang.OurTypeUInt8): raise NotImplementedError(inp, inp.type) + if not isinstance(inp.offset, int): + raise NotImplementedError(inp, inp.offset) + yield from expression(inp.varref) - yield wasm.Statement(inp.type.render() + '.load', 'offset=' + str(inp.member.offset)) + yield wasm.Statement('i32.const', str(inp.offset)) + yield wasm.Statement('call', '$___access_bytes_index___') + return + + if isinstance(inp, ourlang.AccessStructMember): + if isinstance(inp.member.type, ourlang.OurTypeUInt8): + mtyp = 'i32' + else: + # FIXME: Properly implement this + # inp.type.render() is also a hack that doesn't really work consistently + if not isinstance(inp.member.type, ( + ourlang.OurTypeInt32, ourlang.OurTypeFloat32, + ourlang.OurTypeInt64, ourlang.OurTypeFloat64, + )): + raise NotImplementedError + mtyp = inp.member.type.render() + + yield from expression(inp.varref) + yield wasm.Statement(f'{mtyp}.load', 'offset=' + str(inp.member.offset)) return if isinstance(inp, ourlang.AccessTupleMember): @@ -221,7 +252,8 @@ def module(inp: ourlang.Module) -> wasm.Module: result = wasm.Module() result.functions = [ - _generate_allocator(inp), + _generate____new_reference___(inp), + _generate____access_bytes_index___(inp), ] + [ function(x) for x in inp.functions.values() @@ -229,7 +261,7 @@ def module(inp: ourlang.Module) -> wasm.Module: return result -def _generate_allocator(mod: ourlang.Module) -> wasm.Function: +def _generate____new_reference___(mod: ourlang.Module) -> wasm.Function: return wasm.Function( '___new_reference___', True, @@ -252,6 +284,38 @@ def _generate_allocator(mod: ourlang.Module) -> wasm.Function: ], ) +def _generate____access_bytes_index___(mod: ourlang.Module) -> wasm.Function: + return wasm.Function( + '___access_bytes_index___', + False, + [ + ('byt', type_(mod.types['i32']), ), + ('ofs', type_(mod.types['i32']), ), + ], + [ + ], + type_(mod.types['i32']), + [ + wasm.Statement('local.get', '$ofs'), + wasm.Statement('local.get', '$byt'), + wasm.Statement('i32.load'), + wasm.Statement('i32.lt_u'), + + wasm.Statement('if', comment='$ofs < len($byt)'), + wasm.Statement('local.get', '$byt'), + wasm.Statement('i32.const', '4', comment='Leading size field'), + wasm.Statement('i32.add'), + wasm.Statement('local.get', '$ofs'), + wasm.Statement('i32.add'), + wasm.Statement('i32.load8_u', comment='Within bounds'), + wasm.Statement('return'), + wasm.Statement('end'), + + wasm.Statement('i32.const', str(0), comment='Out of bounds'), + wasm.Statement('return'), + ], + ) + def _generate_tuple_constructor(inp: ourlang.TupleConstructor) -> Statements: yield wasm.Statement('i32.const', str(inp.tuple.alloc_size())) yield wasm.Statement('call', '$___new_reference___') @@ -280,16 +344,20 @@ def _generate_struct_constructor(inp: ourlang.StructConstructor) -> Statements: yield wasm.Statement('local.set', '$___new_reference___addr') for member in inp.struct.members: - # FIXME: Properly implement this - # inp.type.render() is also a hack that doesn't really work consistently - if not isinstance(member.type, ( - ourlang.OurTypeInt32, ourlang.OurTypeFloat32, - ourlang.OurTypeInt64, ourlang.OurTypeFloat64, - )): - raise NotImplementedError + if isinstance(member.type, ourlang.OurTypeUInt8): + mtyp = 'i32' + else: + # FIXME: Properly implement this + # inp.type.render() is also a hack that doesn't really work consistently + if not isinstance(member.type, ( + ourlang.OurTypeInt32, ourlang.OurTypeFloat32, + ourlang.OurTypeInt64, ourlang.OurTypeFloat64, + )): + raise NotImplementedError + mtyp = member.type.render() yield wasm.Statement('local.get', '$___new_reference___addr') yield wasm.Statement('local.get', f'${member.name}') - yield wasm.Statement(f'{member.type.render()}.store', 'offset=' + str(member.offset)) + yield wasm.Statement(f'{mtyp}.store', 'offset=' + str(member.offset)) yield wasm.Statement('local.get', '$___new_reference___addr') diff --git a/py2wasm/ourlang.py b/py2wasm/ourlang.py index ba87aec..49398f7 100644 --- a/py2wasm/ourlang.py +++ b/py2wasm/ourlang.py @@ -38,6 +38,18 @@ class OurTypeNone(OurType): def render(self) -> str: return 'None' +class OurTypeUInt8(OurType): + """ + The Integer type, unsigned and 8 bits wide + """ + __slots__ = () + + def render(self) -> str: + return 'u8' + + def alloc_size(self) -> int: + return 4 # Int32 under the hood + class OurTypeInt32(OurType): """ The Integer type, signed and 32 bits wide @@ -86,6 +98,15 @@ class OurTypeFloat64(OurType): def alloc_size(self) -> int: return 8 +class OurTypeBytes(OurType): + """ + The bytes type + """ + __slots__ = () + + def render(self) -> str: + return 'bytes' + class TupleMember: """ Represents a tuple member @@ -146,6 +167,21 @@ class Constant(Expression): """ __slots__ = () +class ConstantUInt8(Constant): + """ + An UInt8 constant value expression within a statement + """ + __slots__ = ('value', ) + + value: int + + def __init__(self, type_: OurTypeUInt8, value: int) -> None: + super().__init__(type_) + self.value = value + + def render(self) -> str: + return str(self.value) + class ConstantInt32(Constant): """ An Int32 constant value expression within a statement @@ -257,7 +293,7 @@ class UnaryOp(Expression): self.right = right def render(self) -> str: - if self.operator in WEBASSEMBLY_BUILDIN_FLOAT_OPS: + if self.operator in WEBASSEMBLY_BUILDIN_FLOAT_OPS or self.operator == 'len': return f'{self.operator}({self.right.render()})' return f'{self.operator}{self.right.render()}' @@ -291,6 +327,24 @@ class FunctionCall(Expression): return f'{self.function.name}({args})' +class AccessBytesIndex(Expression): + """ + Access a bytes index for reading + """ + __slots__ = ('varref', 'offset', ) + + varref: VariableReference + offset: int + + def __init__(self, type_: OurType, varref: VariableReference, offset: int) -> None: + super().__init__(type_) + + self.varref = varref + self.offset = offset + + def render(self) -> str: + return f'{self.varref.render()}[{self.offset}]' + class AccessStructMember(Expression): """ Access a struct member for reading of writing @@ -554,10 +608,12 @@ class Module: def __init__(self) -> None: self.types = { + 'u8': OurTypeUInt8(), 'i32': OurTypeInt32(), 'i64': OurTypeInt64(), 'f32': OurTypeFloat32(), 'f64': OurTypeFloat64(), + 'bytes': OurTypeBytes(), } self.functions = {} self.structs = {} @@ -900,6 +956,18 @@ class OurVisitor: 'sqrt', self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.args[0]), ) + elif node.func.id == 'len': + if not isinstance(exp_type, OurTypeInt32): + _raise_static_error(node, f'Cannot make {node.func.id} result in {exp_type}') + + if 1 != len(node.args): + _raise_static_error(node, f'Function {node.func.id} requires 1 arguments but {len(node.args)} are given') + + return UnaryOp( + exp_type, + 'len', + self.visit_Module_FunctionDef_expr(module, function, our_locals, module.types['bytes'], node.args[0]), + ) else: if node.func.id not in module.functions: _raise_static_error(node, 'Call to undefined function') @@ -966,6 +1034,13 @@ class OurVisitor: _raise_static_error(node, f'Undefined variable {node.value.id}') node_typ = our_locals[node.value.id] + if isinstance(node_typ, OurTypeBytes): + return AccessBytesIndex( + module.types['u8'], + VariableReference(node_typ, node.value.id), + idx, + ) + if not isinstance(node_typ, OurTypeTuple): _raise_static_error(node, f'Cannot take index of non-tuple {node.value.id}') @@ -987,6 +1062,14 @@ class OurVisitor: _not_implemented(node.kind is None, 'Constant.kind') + if isinstance(exp_type, OurTypeUInt8): + if not isinstance(node.value, int): + _raise_static_error(node, 'Expected integer value') + + # FIXME: Range check + + return ConstantUInt8(exp_type, node.value) + if isinstance(exp_type, OurTypeInt32): if not isinstance(node.value, int): _raise_static_error(node, 'Expected integer value') diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index d9f299e..a617c26 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -99,6 +99,15 @@ def _run_pywasm(code_wasm, args): runtime = pywasm.Runtime(module, result.make_imports(), {}) + def set_byte(idx, byt): + runtime.store.mems[0].data[idx] = byt + + args = _convert_bytes_arguments( + args, + lambda x: runtime.exec('___new_reference___', [x]), + set_byte + ) + sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') _dump_memory(runtime.store.mems[0].data) @@ -120,13 +129,22 @@ def _run_pywasm3(code_wasm, args): rtime = env.new_runtime(1024 * 1024) rtime.load(mod) - # sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') - # _dump_memory(rtime.get_memory(0).tobytes()) + def set_byte(idx, byt): + rtime.get_memory(0)[idx] = byt + + args = _convert_bytes_arguments( + args, + rtime.find_function('___new_reference___'), + set_byte + ) + + sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') + _dump_memory(rtime.get_memory(0)) result.returned_value = rtime.find_function('testEntry')(*args) - # sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n') - # _dump_memory(rtime.get_memory(0).tobytes()) + sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n') + _dump_memory(rtime.get_memory(0)) return result @@ -173,6 +191,27 @@ def _run_wasmer(code_wasm, args): return result +def _convert_bytes_arguments(args, new_reference, set_byte): + result = [] + for arg in args: + if not isinstance(arg, bytes): + result.append(arg) + continue + + # TODO: Implement and use the bytes constructor function + offset = new_reference(len(arg) + 4) + result.append(offset) + + # Store the length prefix + for idx, byt in enumerate(len(arg).to_bytes(4, byteorder='little')): + set_byte(offset + idx, byt) + + # Store the actual bytes + for idx, byt in enumerate(arg): + set_byte(offset + 4 + idx, byt) + + return result + def _dump_memory(mem): line_width = 16 diff --git a/tests/integration/test_runtime_checks.py b/tests/integration/test_runtime_checks.py new file mode 100644 index 0000000..abe7081 --- /dev/null +++ b/tests/integration/test_runtime_checks.py @@ -0,0 +1,15 @@ +import pytest + +from .helpers import Suite + +@pytest.mark.integration_test +def test_bytes_index_out_of_bounds(): + code_py = """ +@exported +def testEntry(f: bytes) -> u8: + return f[50] +""" + + result = Suite(code_py, 'test_call').run_code(b'Short', b'Long' * 100) + + assert 0 == result.returned_value diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index ea281e7..f5f28f9 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -3,6 +3,7 @@ import pytest from .helpers import Suite TYPE_MAP = { + 'u8': int, 'i32': int, 'i64': int, 'f32': float, @@ -10,7 +11,7 @@ TYPE_MAP = { } @pytest.mark.integration_test -@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) +@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64', 'u8']) def test_return(type_): code_py = f""" @exported @@ -66,7 +67,7 @@ def testEntry() -> {type_}: assert TYPE_MAP[type_] == type(result.returned_value) @pytest.mark.integration_test -@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) +@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64', 'u8']) def test_arg(type_): code_py = f""" @exported @@ -273,22 +274,23 @@ def testEntry() -> i32: assert [] == result.log_int32_list @pytest.mark.integration_test -def test_struct_0(): - code_py = """ +@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64', 'u8']) +def test_struct_0(type_): + code_py = f""" class CheckedValue: - value: i32 + value: {type_} @exported -def testEntry() -> i32: - return helper(CheckedValue(2345)) +def testEntry() -> {type_}: + return helper(CheckedValue(23)) -def helper(cv: CheckedValue) -> i32: +def helper(cv: CheckedValue) -> {type_}: return cv.value """ result = Suite(code_py, 'test_call').run_code() - assert 2345 == result.returned_value + assert 23 == result.returned_value assert [] == result.log_int32_list @pytest.mark.integration_test @@ -366,6 +368,42 @@ def helper(v: (f32, f32, f32, )) -> f32: assert 3.74 < result.returned_value < 3.75 assert [] == result.log_int32_list +@pytest.mark.integration_test +def test_bytes_address(): + code_py = """ +@exported +def testEntry(f: bytes) -> bytes: + return f +""" + + result = Suite(code_py, 'test_call').run_code(b'This is a test') + + assert 4 == result.returned_value + +@pytest.mark.integration_test +def test_bytes_length(): + code_py = """ +@exported +def testEntry(f: bytes) -> i32: + return len(f) +""" + + result = Suite(code_py, 'test_call').run_code(b'This is another test') + + assert 20 == result.returned_value + +@pytest.mark.integration_test +def test_bytes_index(): + code_py = """ +@exported +def testEntry(f: bytes) -> u8: + return f[8] +""" + + result = Suite(code_py, 'test_call').run_code(b'This is another test') + + assert 0x61 == result.returned_value + @pytest.mark.integration_test @pytest.mark.skip('SIMD support is but a dream') def test_tuple_i32x4(): From 27fa1cc76dedbb96ba397b153ce3bce693a2df59 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 2 Jul 2022 21:48:39 +0200 Subject: [PATCH 39/73] bytes[idx] for any expression idx More steps towards buffer example --- examples/buffer.html | 45 +++++++++++++++++++------------------- examples/buffer.py | 4 ++++ py2wasm/compiler.py | 5 +---- py2wasm/ourlang.py | 51 +++++++++++++++++++++++--------------------- 4 files changed, 54 insertions(+), 51 deletions(-) diff --git a/examples/buffer.html b/examples/buffer.html index c5e13ba..d21c84d 100644 --- a/examples/buffer.html +++ b/examples/buffer.html @@ -1,3 +1,4 @@ + Examples - Buffer @@ -15,38 +16,36 @@ let importObject = {}; let results = document.getElementById('results'); +function log(txt) +{ + let span = document.createElement('span'); + span.innerHTML = txt; + results.appendChild(span); + let br = document.createElement('br'); + results.appendChild(br); +} + WebAssembly.instantiateStreaming(fetch('buffer.wasm'), importObject) .then(app => { // Allocate room within the memory of the WebAssembly class let size = 8; - let offset = app.instance.exports.___new_reference___(4 * size); - var i32 = new Uint32Array(app.instance.exports.memory.buffer, offset, size); + + // TODO: Use bytes constructor + let offset = app.instance.exports.___new_reference___(size); + new Uint32Array(app.instance.exports.memory.buffer, offset, 1)[0] = size; + var i8arr = new Uint8Array(app.instance.exports.memory.buffer, offset + 4, size); + //Fill it let sum = 0; for (var i = 0; i < size; i++) { - i32[i] = i; - sum += i; + i8arr[i] = i + 5; + sum += i8arr[i]; + let from_wasm = app.instance.exports.index(offset, i); + log('i8arr[' + i + '] = ' + from_wasm); } - { - let span = document.createElement('span'); - span.innerHTML = 'expected result = ' + sum; - results.appendChild(span); - let br = document.createElement('br'); - results.appendChild(br); - } - - { - let span = document.createElement('span'); - span.innerHTML = 'actual result = ' + app.instance.exports.sum(offset); - results.appendChild(span); - let br = document.createElement('br'); - results.appendChild(br); - } - - for (var i = 0; i < size; i++) { - console.log(i32[i]); - } + log('expected result = ' + sum); + log('actual result = ' + app.instance.exports.sum(offset)); }); diff --git a/examples/buffer.py b/examples/buffer.py index 6700f6c..74eb023 100644 --- a/examples/buffer.py +++ b/examples/buffer.py @@ -1,3 +1,7 @@ @exported def sum() -> i32: # TODO: Should be [i32] -> i32 return 0 # TODO: Implement me + +@exported +def index(inp: bytes, idx: i32) -> i32: + return inp[idx] diff --git a/py2wasm/compiler.py b/py2wasm/compiler.py index 4b71bae..95f66d7 100644 --- a/py2wasm/compiler.py +++ b/py2wasm/compiler.py @@ -140,11 +140,8 @@ def expression(inp: ourlang.Expression) -> Statements: if not isinstance(inp.type, ourlang.OurTypeUInt8): raise NotImplementedError(inp, inp.type) - if not isinstance(inp.offset, int): - raise NotImplementedError(inp, inp.offset) - yield from expression(inp.varref) - yield wasm.Statement('i32.const', str(inp.offset)) + yield from expression(inp.index) yield wasm.Statement('call', '$___access_bytes_index___') return diff --git a/py2wasm/ourlang.py b/py2wasm/ourlang.py index 49398f7..1ff4ae2 100644 --- a/py2wasm/ourlang.py +++ b/py2wasm/ourlang.py @@ -331,19 +331,19 @@ class AccessBytesIndex(Expression): """ Access a bytes index for reading """ - __slots__ = ('varref', 'offset', ) + __slots__ = ('varref', 'index', ) varref: VariableReference - offset: int + index: Expression - def __init__(self, type_: OurType, varref: VariableReference, offset: int) -> None: + def __init__(self, type_: OurType, varref: VariableReference, index: Expression) -> None: super().__init__(type_) self.varref = varref - self.offset = offset + self.index = index def render(self) -> str: - return f'{self.varref.render()}[{self.offset}]' + return f'{self.varref.render()}[{self.index.render()}]' class AccessStructMember(Expression): """ @@ -1020,13 +1020,6 @@ class OurVisitor: if not isinstance(node.slice, ast.Index): _raise_static_error(node, 'Must subscript using an index') - if not isinstance(node.slice.value, ast.Constant): - _raise_static_error(node, 'Must subscript using a constant index') # FIXME: Implement variable indexes - - idx = node.slice.value.value - if not isinstance(idx, int): - _raise_static_error(node, 'Must subscript using a constant integer index') - if not isinstance(node.ctx, ast.Load): _raise_static_error(node, 'Must be load context') @@ -1034,27 +1027,37 @@ class OurVisitor: _raise_static_error(node, f'Undefined variable {node.value.id}') node_typ = our_locals[node.value.id] + + slice_expr = self.visit_Module_FunctionDef_expr( + module, function, our_locals, module.types['i32'], node.slice.value, + ) + if isinstance(node_typ, OurTypeBytes): return AccessBytesIndex( module.types['u8'], VariableReference(node_typ, node.value.id), - idx, + slice_expr, ) - if not isinstance(node_typ, OurTypeTuple): - _raise_static_error(node, f'Cannot take index of non-tuple {node.value.id}') + if isinstance(node_typ, OurTypeTuple): + if not isinstance(slice_expr, ConstantInt32): + _raise_static_error(node, 'Must subscript using a constant index') - if len(node_typ.members) <= idx: - _raise_static_error(node, f'Index {idx} out of bounds for tuple {node.value.id}') + idx = slice_expr.value - member = node_typ.members[idx] - if exp_type != member.type: - _raise_static_error(node, f'Expected {exp_type.render()}, {node.value.id}[{idx}] is actually {member.type.render()}') + if len(node_typ.members) <= idx: + _raise_static_error(node, f'Index {idx} out of bounds for tuple {node.value.id}') - return AccessTupleMember( - VariableReference(node_typ, node.value.id), - member, - ) + member = node_typ.members[idx] + if exp_type != member.type: + _raise_static_error(node, f'Expected {exp_type.render()}, {node.value.id}[{idx}] is actually {member.type.render()}') + + return AccessTupleMember( + VariableReference(node_typ, node.value.id), + member, + ) + + _raise_static_error(node, f'Cannot take index of {node_typ.render()} {node.value.id}') def visit_Module_FunctionDef_Constant(self, module: Module, function: Function, exp_type: OurType, node: ast.Constant) -> Expression: del module From 17aa5fd6f9169ed225578cfcb2641ceafc5cb2cc Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Fri, 8 Jul 2022 20:23:12 +0200 Subject: [PATCH 40/73] Examples HTML now serve higlighted WAT --- Makefile | 5 ++++- examples/.gitignore | 1 + examples/buffer.html | 2 +- examples/fib.html | 2 +- requirements.txt | 1 + 5 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 7ccbb41..1240788 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,9 @@ WASM2C := $(WABT_DIR)/bin/wasm2c %.wat: %.py py2wasm/*.py venv/.done venv/bin/python -m py2wasm $< $@ +%.wat.html: %.wat + venv/bin/pygmentize -l wat -O full -f html $^ -o $@ + %.wasm: %.wat $(WAT2WASM) $^ -o $@ @@ -15,7 +18,7 @@ WASM2C := $(WABT_DIR)/bin/wasm2c # %.exe: %.c # cc $^ -o $@ -I $(WABT_DIR)/wasm2c -examples: venv/.done $(subst .py,.wasm,$(wildcard examples/*.py)) +examples: venv/.done $(subst .py,.wasm,$(wildcard examples/*.py)) $(subst .py,.wat.html,$(wildcard examples/*.py)) venv/bin/python3 -m http.server --directory examples test: venv/.done diff --git a/examples/.gitignore b/examples/.gitignore index 50765be..6edc9f7 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -1,2 +1,3 @@ *.wasm *.wat +*.wat.html diff --git a/examples/buffer.html b/examples/buffer.html index d21c84d..294ba0f 100644 --- a/examples/buffer.html +++ b/examples/buffer.html @@ -6,7 +6,7 @@

Buffer

-List - Source - WebAssembly +List - Source - WebAssembly
diff --git a/examples/fib.html b/examples/fib.html index 8d62880..865c5a1 100644 --- a/examples/fib.html +++ b/examples/fib.html @@ -5,7 +5,7 @@

Fibonacci

-List - Source - WebAssembly +List - Source - WebAssembly
diff --git a/requirements.txt b/requirements.txt index b969a61..d29e53c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ mypy==0.812 +pygments==2.12.0 pylint==2.7.4 pytest==6.2.2 pytest-integration==0.2.2 From eb74c8770d83f66bc12c7312366766501630917a Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Fri, 8 Jul 2022 20:41:58 +0200 Subject: [PATCH 41/73] Examples HTML now serve higlighted py --- Makefile | 5 ++++- examples/.gitignore | 1 + examples/buffer.html | 2 +- examples/fib.html | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 1240788..49045bf 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,9 @@ WASM2C := $(WABT_DIR)/bin/wasm2c %.wat.html: %.wat venv/bin/pygmentize -l wat -O full -f html $^ -o $@ +%.py.html: %.py + venv/bin/pygmentize -l py -O full -f html $^ -o $@ + %.wasm: %.wat $(WAT2WASM) $^ -o $@ @@ -18,7 +21,7 @@ WASM2C := $(WABT_DIR)/bin/wasm2c # %.exe: %.c # cc $^ -o $@ -I $(WABT_DIR)/wasm2c -examples: venv/.done $(subst .py,.wasm,$(wildcard examples/*.py)) $(subst .py,.wat.html,$(wildcard examples/*.py)) +examples: venv/.done $(subst .py,.wasm,$(wildcard examples/*.py)) $(subst .py,.wat.html,$(wildcard examples/*.py)) $(subst .py,.py.html,$(wildcard examples/*.py)) venv/bin/python3 -m http.server --directory examples test: venv/.done diff --git a/examples/.gitignore b/examples/.gitignore index 6edc9f7..de726e0 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -1,3 +1,4 @@ +*.py.html *.wasm *.wat *.wat.html diff --git a/examples/buffer.html b/examples/buffer.html index 294ba0f..d0b14ca 100644 --- a/examples/buffer.html +++ b/examples/buffer.html @@ -6,7 +6,7 @@

Buffer

-List - Source - WebAssembly +List - Source - WebAssembly
diff --git a/examples/fib.html b/examples/fib.html index 865c5a1..7a2bccb 100644 --- a/examples/fib.html +++ b/examples/fib.html @@ -5,7 +5,7 @@

Fibonacci

-List - Source - WebAssembly +List - Source - WebAssembly
From 76d80f57cbd2e21050a74670ccb5bef6e041ea73 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Fri, 8 Jul 2022 21:06:13 +0200 Subject: [PATCH 42/73] Imports --- examples/imported.html | 44 ++++++++++++++++++++++++++++++++ examples/imported.py | 7 +++++ examples/index.html | 1 + py2wasm/compiler.py | 29 +++++++++++++++++++++ py2wasm/ourlang.py | 29 ++++++++++++++++++--- py2wasm/wasm.py | 14 +++++++--- tests/integration/helpers.py | 23 ++++++++++++++--- tests/integration/test_simple.py | 24 +++++++++++++++++ 8 files changed, 159 insertions(+), 12 deletions(-) create mode 100644 examples/imported.html create mode 100644 examples/imported.py diff --git a/examples/imported.html b/examples/imported.html new file mode 100644 index 0000000..a69c02e --- /dev/null +++ b/examples/imported.html @@ -0,0 +1,44 @@ + + + +Examples - Imported + + +

Imported

+ +List - Source - WebAssembly + +
+ + + + + + diff --git a/examples/imported.py b/examples/imported.py new file mode 100644 index 0000000..f5c2663 --- /dev/null +++ b/examples/imported.py @@ -0,0 +1,7 @@ +@imported +def log(no: i32) -> None: + pass + +@exported +def run(a: i32, b: i32) -> None: + return log(a * b) diff --git a/examples/index.html b/examples/index.html index 29e73c8..46e1fc1 100644 --- a/examples/index.html +++ b/examples/index.html @@ -11,6 +11,7 @@

Technical

diff --git a/py2wasm/compiler.py b/py2wasm/compiler.py index 95f66d7..3605ba5 100644 --- a/py2wasm/compiler.py +++ b/py2wasm/compiler.py @@ -9,6 +9,9 @@ from . import wasm Statements = Generator[wasm.Statement, None, None] def type_(inp: ourlang.OurType) -> wasm.OurType: + if isinstance(inp, ourlang.OurTypeNone): + return wasm.OurTypeNone() + if isinstance(inp, ourlang.OurTypeUInt8): # WebAssembly has only support for 32 and 64 bits # So we need to store more memory per byte @@ -205,12 +208,31 @@ def statement(inp: ourlang.Statement) -> Statements: yield from statement_if(inp) return + if isinstance(inp, ourlang.StatementPass): + return + raise NotImplementedError(statement, inp) def function_argument(inp: Tuple[str, ourlang.OurType]) -> wasm.Param: return (inp[0], type_(inp[1]), ) +def import_(inp: ourlang.Function) -> wasm.Import: + assert inp.imported + + return wasm.Import( + 'imports', + inp.name, + inp.name, + [ + function_argument(x) + for x in inp.posonlyargs + ], + type_(inp.returns) + ) + def function(inp: ourlang.Function) -> wasm.Function: + assert not inp.imported + if isinstance(inp, ourlang.TupleConstructor): statements = [ *_generate_tuple_constructor(inp) @@ -248,12 +270,19 @@ def function(inp: ourlang.Function) -> wasm.Function: def module(inp: ourlang.Module) -> wasm.Module: result = wasm.Module() + result.imports = [ + import_(x) + for x in inp.functions.values() + if x.imported + ] + result.functions = [ _generate____new_reference___(inp), _generate____access_bytes_index___(inp), ] + [ function(x) for x in inp.functions.values() + if not x.imported ] return result diff --git a/py2wasm/ourlang.py b/py2wasm/ourlang.py index 1ff4ae2..303d94d 100644 --- a/py2wasm/ourlang.py +++ b/py2wasm/ourlang.py @@ -458,11 +458,12 @@ class Function: """ A function processes input and produces output """ - __slots__ = ('name', 'lineno', 'exported', 'buildin', 'statements', 'returns', 'posonlyargs', ) + __slots__ = ('name', 'lineno', 'exported', 'imported', 'buildin', 'statements', 'returns', 'posonlyargs', ) name: str lineno: int exported: bool + imported: bool buildin: bool statements: List[Statement] returns: OurType @@ -472,6 +473,7 @@ class Function: self.name = name self.lineno = lineno self.exported = False + self.imported = False self.buildin = False self.statements = [] self.returns = OurTypeNone() @@ -483,9 +485,14 @@ class Function: This'll look like Python code. """ + statements = self.statements + result = '' if self.exported: result += '@exported\n' + if self.imported: + result += '@imported\n' + statements = [StatementPass()] args = ', '.join( f'{x}: {y.render()}' @@ -493,7 +500,7 @@ class Function: ) result += f'def {self.name}({args}) -> {self.returns.render()}:\n' - for stmt in self.statements: + for stmt in statements: for line in stmt.render(): result += f' {line}\n' if line else '\n' return result @@ -608,6 +615,7 @@ class Module: def __init__(self) -> None: self.types = { + 'None': OurTypeNone(), 'u8': OurTypeUInt8(), 'i32': OurTypeInt32(), 'i64': OurTypeInt64(), @@ -730,8 +738,12 @@ class OurVisitor: _raise_static_error(decorator, 'Function decorators must be string') if not isinstance(decorator.ctx, ast.Load): _raise_static_error(decorator, 'Must be load context') - _not_implemented(decorator.id != 'exports', 'Custom decorators') - function.exported = True + _not_implemented(decorator.id in ('exported', 'imported'), 'Custom decorators') + + if decorator.id == 'exported': + function.exported = True + else: + function.imported = True if node.returns: function.returns = self.visit_type(module, node.returns) @@ -816,6 +828,9 @@ class OurVisitor: return result + if isinstance(node, ast.Pass): + return StatementPass() + raise NotImplementedError(f'{node} as stmt in FunctionDef') def visit_Module_FunctionDef_expr(self, module: Module, function: Function, our_locals: OurLocals, exp_type: OurType, node: ast.expr) -> Expression: @@ -1108,6 +1123,12 @@ class OurVisitor: raise NotImplementedError(f'{node} as const for type {exp_type.render()}') def visit_type(self, module: Module, node: ast.expr) -> OurType: + if isinstance(node, ast.Constant): + if node.value is None: + return module.types['None'] + + _raise_static_error(node, f'Unrecognized type {node.value}') + if isinstance(node, ast.Name): if not isinstance(node.ctx, ast.Load): _raise_static_error(node, 'Must be load context') diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py index 049df62..a4c274a 100644 --- a/py2wasm/wasm.py +++ b/py2wasm/wasm.py @@ -156,22 +156,28 @@ class Import: name: str, intname: str, params: Iterable[Param], + result: OurType, ) -> None: self.module = module self.name = name self.intname = intname self.params = [*params] - self.result: OurType = OurTypeNone() + self.result = result def generate(self) -> str: """ Generates the text version """ - return '(import "{}" "{}" (func ${}{}))'.format( + return '(import "{}" "{}" (func ${}{}{}))'.format( self.module, self.name, self.intname, - ''.join(' (param {})'.format(x[1]) for x in self.params) + ''.join( + f' (param {typ.to_wasm()})' + for _, typ in self.params + ), + '' if isinstance(self.result, OurTypeNone) + else f' (result {self.result.to_wasm()})' ) class Statement: @@ -263,7 +269,7 @@ class Module: Generates the text version """ return '(module\n {}\n {}\n {})\n'.format( - self.memory.generate(), '\n '.join(x.generate() for x in self.imports), + self.memory.generate(), '\n '.join(x.generate() for x in self.functions), ) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index a617c26..af90b92 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -63,7 +63,7 @@ class Suite: self.code_py = code_py self.test_name = test_name - def run_code(self, *args): + def run_code(self, *args, runtime='pywasm3', imports=None): """ Compiles the given python code into wasm and then runs it @@ -89,7 +89,16 @@ class Suite: code_wasm = wat2wasm(code_wat) # Run assembly code - return _run_pywasm3(code_wasm, args) + if 'pywasm' == runtime: + return _run_pywasm(code_wasm, args) + if 'pywasm3' == runtime: + return _run_pywasm3(code_wasm, args) + if 'wasmtime' == runtime: + return _run_wasmtime(code_wasm, args) + if 'wasmer' == runtime: + return _run_wasmer(code_wasm, args, imports) + + raise Exception(f'Invalid runtime: {runtime}') def _run_pywasm(code_wasm, args): # https://pypi.org/project/pywasm/ @@ -168,17 +177,23 @@ def _run_wasmtime(code_wasm, args): return result -def _run_wasmer(code_wasm, args): +def _run_wasmer(code_wasm, args, imports): # https://pypi.org/project/wasmer/ result = SuiteResult() store = wasmer.Store(wasmer.engine.JIT(wasmer_compiler_cranelift.Compiler)) + import_object = wasmer.ImportObject() + import_object.register('imports', { + k: wasmer.Function(store, v) + for k, v in (imports or {}).items() + }) + # Let's compile the module to be able to execute it! module = wasmer.Module(store, code_wasm) # Now the module is compiled, we can instantiate it. - instance = wasmer.Instance(module) + instance = wasmer.Instance(module, import_object) sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') sys.stderr.write('\n') diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index f5f28f9..b8f5bd3 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -416,3 +416,27 @@ def testEntry() -> i32x4: result = Suite(code_py, 'test_rgb2hsl').run_code() assert (1, 2, 3, 0) == result.returned_value + +@pytest.mark.integration_test +def test_imported(): + code_py = """ +@imported +def helper() -> i32: + pass + +@exported +def testEntry() -> i32: + return helper() +""" + + def helper() -> int: + return 4238 + + result = Suite(code_py, 'test_imported').run_code( + runtime='wasmer', + imports={ + 'helper': helper, + } + ) + + assert 4238 == result.returned_value From c181c61040b84e19404edc7b1c6e2cca24be0475 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 9 Jul 2022 11:34:20 +0200 Subject: [PATCH 43/73] Extended the import example a bit for sanity's sake --- tests/integration/test_simple.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index b8f5bd3..9bb3139 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -421,16 +421,16 @@ def testEntry() -> i32x4: def test_imported(): code_py = """ @imported -def helper() -> i32: +def helper(mul: i32) -> i32: pass @exported def testEntry() -> i32: - return helper() + return helper(2) """ - def helper() -> int: - return 4238 + def helper(mul: int) -> int: + return 4238 * mul result = Suite(code_py, 'test_imported').run_code( runtime='wasmer', @@ -439,4 +439,4 @@ def testEntry() -> i32: } ) - assert 4238 == result.returned_value + assert 8476 == result.returned_value From 14eede6b066034ef66f4ebe4f0d402984be03ae2 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 9 Jul 2022 12:30:28 +0200 Subject: [PATCH 44/73] Cleanup to wasm.py --- py2wasm/__main__.py | 4 +- py2wasm/compiler.py | 26 ++-- py2wasm/ourlang.py | 4 +- py2wasm/wasm.py | 249 +++++++++++++---------------------- tests/integration/helpers.py | 7 +- 5 files changed, 109 insertions(+), 181 deletions(-) diff --git a/py2wasm/__main__.py b/py2wasm/__main__.py index f66b313..7c65724 100644 --- a/py2wasm/__main__.py +++ b/py2wasm/__main__.py @@ -16,8 +16,8 @@ def main(source: str, sink: str) -> int: code_py = fil.read() our_module = our_process(code_py, source) - wat_module = module(our_module) - code_wat = wat_module.generate() + wasm_module = module(our_module) + code_wat = wasm_module.to_wat() with open(sink, 'w') as fil: fil.write(code_wat) diff --git a/py2wasm/compiler.py b/py2wasm/compiler.py index 3605ba5..cf2d99c 100644 --- a/py2wasm/compiler.py +++ b/py2wasm/compiler.py @@ -8,31 +8,31 @@ from . import wasm Statements = Generator[wasm.Statement, None, None] -def type_(inp: ourlang.OurType) -> wasm.OurType: +def type_(inp: ourlang.OurType) -> wasm.WasmType: if isinstance(inp, ourlang.OurTypeNone): - return wasm.OurTypeNone() + return wasm.WasmTypeNone() if isinstance(inp, ourlang.OurTypeUInt8): # WebAssembly has only support for 32 and 64 bits # So we need to store more memory per byte - return wasm.OurTypeInt32() + return wasm.WasmTypeInt32() if isinstance(inp, ourlang.OurTypeInt32): - return wasm.OurTypeInt32() + return wasm.WasmTypeInt32() if isinstance(inp, ourlang.OurTypeInt64): - return wasm.OurTypeInt64() + return wasm.WasmTypeInt64() if isinstance(inp, ourlang.OurTypeFloat32): - return wasm.OurTypeFloat32() + return wasm.WasmTypeFloat32() if isinstance(inp, ourlang.OurTypeFloat64): - return wasm.OurTypeFloat64() + return wasm.WasmTypeFloat64() if isinstance(inp, (ourlang.Struct, ourlang.OurTypeTuple, ourlang.OurTypeBytes)): # Structs and tuples are passed as pointer # And pointers are i32 - return wasm.OurTypeInt32() + return wasm.WasmTypeInt32() raise NotImplementedError(type_, inp) @@ -238,14 +238,14 @@ def function(inp: ourlang.Function) -> wasm.Function: *_generate_tuple_constructor(inp) ] locals_ = [ - ('___new_reference___addr', wasm.OurTypeInt32(), ), + ('___new_reference___addr', wasm.WasmTypeInt32(), ), ] elif isinstance(inp, ourlang.StructConstructor): statements = [ *_generate_struct_constructor(inp) ] locals_ = [ - ('___new_reference___addr', wasm.OurTypeInt32(), ), + ('___new_reference___addr', wasm.WasmTypeInt32(), ), ] else: statements = [ @@ -257,7 +257,7 @@ def function(inp: ourlang.Function) -> wasm.Function: return wasm.Function( inp.name, - inp.exported, + inp.name if inp.exported else None, [ function_argument(x) for x in inp.posonlyargs @@ -290,7 +290,7 @@ def module(inp: ourlang.Module) -> wasm.Module: def _generate____new_reference___(mod: ourlang.Module) -> wasm.Function: return wasm.Function( '___new_reference___', - True, + '___new_reference___', [ ('alloc_size', type_(mod.types['i32']), ), ], @@ -313,7 +313,7 @@ def _generate____new_reference___(mod: ourlang.Module) -> wasm.Function: def _generate____access_bytes_index___(mod: ourlang.Module) -> wasm.Function: return wasm.Function( '___access_bytes_index___', - False, + None, [ ('byt', type_(mod.types['i32']), ), ('ofs', type_(mod.types['i32']), ), diff --git a/py2wasm/ourlang.py b/py2wasm/ourlang.py index 303d94d..99156be 100644 --- a/py2wasm/ourlang.py +++ b/py2wasm/ourlang.py @@ -458,13 +458,12 @@ class Function: """ A function processes input and produces output """ - __slots__ = ('name', 'lineno', 'exported', 'imported', 'buildin', 'statements', 'returns', 'posonlyargs', ) + __slots__ = ('name', 'lineno', 'exported', 'imported', 'statements', 'returns', 'posonlyargs', ) name: str lineno: int exported: bool imported: bool - buildin: bool statements: List[Statement] returns: OurType posonlyargs: List[Tuple[str, OurType]] @@ -474,7 +473,6 @@ class Function: self.lineno = lineno self.exported = False self.imported = False - self.buildin = False self.statements = [] self.returns = OurTypeNone() self.posonlyargs = [] diff --git a/py2wasm/wasm.py b/py2wasm/wasm.py index a4c274a..3e5cff4 100644 --- a/py2wasm/wasm.py +++ b/py2wasm/wasm.py @@ -3,150 +3,77 @@ Python classes for storing the representation of Web Assembly code, and being able to conver it to Web Assembly Text Format """ -from typing import Any, Iterable, List, Optional, Tuple, Union +from typing import Iterable, List, Optional, Tuple -### -### This part is more intermediate code -### +class WatSerializable: + """ + Mixin for clases that can be serialized as WebAssembly Text + """ + def to_wat(self) -> str: + """ + Renders this object as WebAssembly Text + """ + raise NotImplementedError(self, 'to_wat') -class OurType: - def to_python(self) -> str: - raise NotImplementedError(self, 'to_python') +class WasmType(WatSerializable): + """ + Type base class + """ - def to_wasm(self) -> str: - raise NotImplementedError(self, 'to_wasm') +class WasmTypeNone(WasmType): + """ + Type when there is no type + """ + def to_wat(self) -> str: + raise Exception('None type is only a placeholder') - def alloc_size(self) -> int: - raise NotImplementedError(self, 'alloc_size') +class WasmTypeInt32(WasmType): + """ + i32 value -class OurTypeNone(OurType): - pass - -class OurTypeBool(OurType): - pass - -class OurTypeInt32(OurType): - def to_python(self) -> str: + Signed or not depends on the operations, not the type + """ + def to_wat(self) -> str: return 'i32' - def to_wasm(self) -> str: - return 'i32' +class WasmTypeInt64(WasmType): + """ + i64 value - def alloc_size(self) -> int: - return 4 - -class OurTypeInt64(OurType): - def to_wasm(self) -> str: + Signed or not depends on the operations, not the type + """ + def to_wat(self) -> str: return 'i64' - def alloc_size(self) -> int: - return 8 - -class OurTypeFloat32(OurType): - def to_wasm(self) -> str: +class WasmTypeFloat32(WasmType): + """ + f32 value + """ + def to_wat(self) -> str: return 'f32' - def alloc_size(self) -> int: - return 4 - -class OurTypeFloat64(OurType): - def to_wasm(self) -> str: +class WasmTypeFloat64(WasmType): + """ + f64 value + """ + def to_wat(self) -> str: return 'f64' - def alloc_size(self) -> int: - return 8 - -class OurTypeVector(OurType): +class WasmTypeVector(WasmType): """ A vector is a 128-bit value """ - def to_wasm(self) -> str: + def to_wat(self) -> str: return 'v128' - def alloc_size(self) -> int: - return 16 - -class OurTypeVectorInt32x4(OurTypeVector): +class WasmTypeVectorInt32x4(WasmTypeVector): """ 4 Int32 values in a single vector """ -class Constant: - """ - TODO - """ - def __init__(self, value: Union[None, bool, int, float]) -> None: - self.value = value +Param = Tuple[str, WasmType] -class ClassMember: - """ - Represents a class member - """ - def __init__(self, name: str, type_: OurType, offset: int, default: Optional[Constant]) -> None: - self.name = name - self.type = type_ - self.offset = offset - self.default = default - -class OurTypeClass(OurType): - """ - Represents a class - """ - def __init__(self, name: str, members: List[ClassMember]) -> None: - self.name = name - self.members = members - - def to_wasm(self) -> str: - return 'i32' # WASM uses 32 bit pointers - - def alloc_size(self) -> int: - return sum(x.type.alloc_size() for x in self.members) - -class TupleMember: - """ - Represents a tuple member - """ - def __init__(self, type_: OurType, offset: int) -> None: - self.type = type_ - self.offset = offset - -class OurTypeTuple(OurType): - """ - Represents a tuple - """ - def __init__(self, members: List[TupleMember]) -> None: - self.members = members - - def to_python(self) -> str: - return 'Tuple[' + ( - ', '.join(x.type.to_python() for x in self.members) - ) + ']' - - def to_wasm(self) -> str: - return 'i32' # WASM uses 32 bit pointers - - def alloc_size(self) -> int: - return sum(x.type.alloc_size() for x in self.members) - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, OurTypeTuple): - raise NotImplementedError - - return ( - len(self.members) == len(other.members) - and all( - self.members[x].type == other.members[x].type - for x in range(len(self.members)) - ) - ) - -Param = Tuple[str, OurType] - -### -## This part is more actual web assembly -### - -class Import: +class Import(WatSerializable): """ Represents a Web Assembly import """ @@ -156,7 +83,7 @@ class Import: name: str, intname: str, params: Iterable[Param], - result: OurType, + result: WasmType, ) -> None: self.module = module self.name = name @@ -164,23 +91,20 @@ class Import: self.params = [*params] self.result = result - def generate(self) -> str: - """ - Generates the text version - """ + def to_wat(self) -> str: return '(import "{}" "{}" (func ${}{}{}))'.format( self.module, self.name, self.intname, ''.join( - f' (param {typ.to_wasm()})' + f' (param {typ.to_wat()})' for _, typ in self.params ), - '' if isinstance(self.result, OurTypeNone) - else f' (result {self.result.to_wasm()})' + '' if isinstance(self.result, WasmTypeNone) + else f' (result {self.result.to_wat()})' ) -class Statement: +class Statement(WatSerializable): """ Represents a Web Assembly statement """ @@ -189,73 +113,76 @@ class Statement: self.args = args self.comment = comment - def generate(self) -> str: - """ - Generates the text version - """ + def to_wat(self) -> str: args = ' '.join(self.args) comment = f' ;; {self.comment}' if self.comment else '' return f'{self.name} {args}{comment}' -class Function: +class Function(WatSerializable): """ Represents a Web Assembly function """ def __init__( self, name: str, - exported: bool, + exported_name: Optional[str], params: Iterable[Param], locals_: Iterable[Param], - result: OurType, + result: WasmType, statements: Iterable[Statement], ) -> None: self.name = name - self.exported = exported + self.exported_name = exported_name self.params = [*params] self.locals = [*locals_] self.result = result self.statements = [*statements] - def generate(self) -> str: - """ - Generates the text version - """ + def to_wat(self) -> str: header = f'${self.name}' # Name for internal use - if self.exported: + if self.exported_name is not None: # Name for external use - # TODO: Consider: Make exported('export_name') work - header += f' (export "{self.name}")' + header += f' (export "{self.exported_name}")' for nam, typ in self.params: - header += f' (param ${nam} {typ.to_wasm()})' + header += f' (param ${nam} {typ.to_wat()})' - if not isinstance(self.result, OurTypeNone): - header += f' (result {self.result.to_wasm()})' + if not isinstance(self.result, WasmTypeNone): + header += f' (result {self.result.to_wat()})' for nam, typ in self.locals: - header += f' (local ${nam} {typ.to_wasm()})' + header += f' (local ${nam} {typ.to_wat()})' return '(func {}\n {}\n )'.format( header, - '\n '.join(x.generate() for x in self.statements), + '\n '.join(x.to_wat() for x in self.statements), ) -class ModuleMemory: +class ModuleMemory(WatSerializable): + """ + Represents a WebAssembly module's memory + """ def __init__(self, data: bytes = b'') -> None: self.data = data - def generate(self) -> str: - return '(memory 1)\n (data (memory 0) (i32.const 0) "{}")\n (export "memory" (memory 0))\n'.format( - ''.join( - f'\\{x:02x}' - for x in self.data - ) + def to_wat(self) -> str: + """ + Renders this memory as WebAssembly Text + """ + data = ''.join( + f'\\{x:02x}' + for x in self.data ) -class Module: + return ( + '(memory 1)\n ' + f'(data (memory 0) (i32.const 0) "{data}")\n ' + '(export "memory" (memory 0))\n' + ) + +class Module(WatSerializable): """ Represents a Web Assembly module """ @@ -264,12 +191,12 @@ class Module: self.functions: List[Function] = [] self.memory = ModuleMemory(b'\x04') # For ___new_reference___ - def generate(self) -> str: + def to_wat(self) -> str: """ Generates the text version """ return '(module\n {}\n {}\n {})\n'.format( - '\n '.join(x.generate() for x in self.imports), - self.memory.generate(), - '\n '.join(x.generate() for x in self.functions), + '\n '.join(x.to_wat() for x in self.imports), + self.memory.to_wat(), + '\n '.join(x.to_wat() for x in self.functions), ) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index af90b92..c1ec6a7 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -59,6 +59,9 @@ class SuiteResult: } class Suite: + """ + WebAssembly test suite + """ def __init__(self, code_py, test_name): self.code_py = code_py self.test_name = test_name @@ -76,10 +79,10 @@ class Suite: assert self.code_py == '\n' + our_module.render() # \n for formatting in tests # Compile - wat_module = module(our_module) + wasm_module = module(our_module) # Render as text - code_wat = wat_module.generate() + code_wat = wasm_module.to_wat() sys.stderr.write(f'{DASHES} Assembly {DASHES}\n') From d32613d9b8cfddd57f950d1ff86a98753374643d Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 9 Jul 2022 12:35:32 +0200 Subject: [PATCH 45/73] We have a name \o/ --- Makefile | 8 ++++---- README.md | 15 +++++++++++++-- {py2wasm => phasm}/__init__.py | 0 {py2wasm => phasm}/__main__.py | 0 {py2wasm => phasm}/compiler.py | 0 {py2wasm => phasm}/ourlang.py | 0 {py2wasm => phasm}/utils.py | 0 {py2wasm => phasm}/wasm.py | 0 tests/integration/helpers.py | 4 ++-- tests/integration/test_static_checking.py | 5 ++--- 10 files changed, 21 insertions(+), 11 deletions(-) rename {py2wasm => phasm}/__init__.py (100%) rename {py2wasm => phasm}/__main__.py (100%) rename {py2wasm => phasm}/compiler.py (100%) rename {py2wasm => phasm}/ourlang.py (100%) rename {py2wasm => phasm}/utils.py (100%) rename {py2wasm => phasm}/wasm.py (100%) diff --git a/Makefile b/Makefile index 49045bf..80cc868 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,8 @@ WABT_DIR := /home/johan/Sources/github.com/WebAssembly/wabt WAT2WASM := $(WABT_DIR)/bin/wat2wasm WASM2C := $(WABT_DIR)/bin/wasm2c -%.wat: %.py py2wasm/*.py venv/.done - venv/bin/python -m py2wasm $< $@ +%.wat: %.py phasm/*.py venv/.done + venv/bin/python -m phasm $< $@ %.wat.html: %.wat venv/bin/pygmentize -l wat -O full -f html $^ -o $@ @@ -28,10 +28,10 @@ test: venv/.done WAT2WASM=$(WAT2WASM) venv/bin/pytest tests $(TEST_FLAGS) lint: venv/.done - venv/bin/pylint py2wasm + venv/bin/pylint phasm typecheck: venv/.done - venv/bin/mypy --strict py2wasm + venv/bin/mypy --strict phasm venv/.done: requirements.txt python3.8 -m venv venv diff --git a/README.md b/README.md index 93683c4..5ab49f7 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,16 @@ -webasm (python) -=============== +phasm +===== + +Elevator pitch +-------------- +A language that looks like Python, handles like Haskell, and compiles to +WebAssembly. + +Naming +------ +- p from python +- ha from Haskell +- asm from WebAssembly You will need wat2wasm from github.com/WebAssembly/wabt in your path. diff --git a/py2wasm/__init__.py b/phasm/__init__.py similarity index 100% rename from py2wasm/__init__.py rename to phasm/__init__.py diff --git a/py2wasm/__main__.py b/phasm/__main__.py similarity index 100% rename from py2wasm/__main__.py rename to phasm/__main__.py diff --git a/py2wasm/compiler.py b/phasm/compiler.py similarity index 100% rename from py2wasm/compiler.py rename to phasm/compiler.py diff --git a/py2wasm/ourlang.py b/phasm/ourlang.py similarity index 100% rename from py2wasm/ourlang.py rename to phasm/ourlang.py diff --git a/py2wasm/utils.py b/phasm/utils.py similarity index 100% rename from py2wasm/utils.py rename to phasm/utils.py diff --git a/py2wasm/wasm.py b/phasm/wasm.py similarity index 100% rename from py2wasm/wasm.py rename to phasm/wasm.py diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index c1ec6a7..d19c697 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -14,8 +14,8 @@ import wasmer_compiler_cranelift import wasmtime -from py2wasm.utils import our_process -from py2wasm.compiler import module +from phasm.utils import our_process +from phasm.compiler import module DASHES = '-' * 16 diff --git a/tests/integration/test_static_checking.py b/tests/integration/test_static_checking.py index f3de90b..ad43a7c 100644 --- a/tests/integration/test_static_checking.py +++ b/tests/integration/test_static_checking.py @@ -1,8 +1,7 @@ import pytest -from py2wasm.utils import our_process - -from py2wasm.ourlang import StaticError +from phasm.utils import our_process +from phasm.ourlang import StaticError @pytest.mark.integration_test @pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) From cc762cfa440c2bfa2de0fb02f701daadb89b3502 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 9 Jul 2022 12:48:54 +0200 Subject: [PATCH 46/73] Typing is a chapter of its own --- phasm/compiler.py | 57 ++++----- phasm/ourlang.py | 315 ++++++++++------------------------------------ phasm/typing.py | 206 ++++++++++++++++++++++++++++++ 3 files changed, 302 insertions(+), 276 deletions(-) create mode 100644 phasm/typing.py diff --git a/phasm/compiler.py b/phasm/compiler.py index cf2d99c..979543e 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -4,32 +4,33 @@ This module contains the code to convert parsed Ourlang into WebAssembly code from typing import Generator, Tuple from . import ourlang +from . import typing from . import wasm Statements = Generator[wasm.Statement, None, None] -def type_(inp: ourlang.OurType) -> wasm.WasmType: - if isinstance(inp, ourlang.OurTypeNone): +def type_(inp: typing.TypeBase) -> wasm.WasmType: + if isinstance(inp, typing.TypeNone): return wasm.WasmTypeNone() - if isinstance(inp, ourlang.OurTypeUInt8): + if isinstance(inp, typing.TypeUInt8): # WebAssembly has only support for 32 and 64 bits # So we need to store more memory per byte return wasm.WasmTypeInt32() - if isinstance(inp, ourlang.OurTypeInt32): + if isinstance(inp, typing.TypeInt32): return wasm.WasmTypeInt32() - if isinstance(inp, ourlang.OurTypeInt64): + if isinstance(inp, typing.TypeInt64): return wasm.WasmTypeInt64() - if isinstance(inp, ourlang.OurTypeFloat32): + if isinstance(inp, typing.TypeFloat32): return wasm.WasmTypeFloat32() - if isinstance(inp, ourlang.OurTypeFloat64): + if isinstance(inp, typing.TypeFloat64): return wasm.WasmTypeFloat64() - if isinstance(inp, (ourlang.Struct, ourlang.OurTypeTuple, ourlang.OurTypeBytes)): + if isinstance(inp, (typing.TypeStruct, typing.TypeTuple, typing.TypeBytes)): # Structs and tuples are passed as pointer # And pointers are i32 return wasm.WasmTypeInt32() @@ -87,25 +88,25 @@ def expression(inp: ourlang.Expression) -> Statements: yield from expression(inp.left) yield from expression(inp.right) - if isinstance(inp.type, ourlang.OurTypeInt32): + if isinstance(inp.type, typing.TypeInt32): if operator := OPERATOR_MAP.get(inp.operator, None): yield wasm.Statement(f'i32.{operator}') return if operator := I32_OPERATOR_MAP.get(inp.operator, None): yield wasm.Statement(f'i32.{operator}') return - if isinstance(inp.type, ourlang.OurTypeInt64): + if isinstance(inp.type, typing.TypeInt64): if operator := OPERATOR_MAP.get(inp.operator, None): yield wasm.Statement(f'i64.{operator}') return if operator := I64_OPERATOR_MAP.get(inp.operator, None): yield wasm.Statement(f'i64.{operator}') return - if isinstance(inp.type, ourlang.OurTypeFloat32): + if isinstance(inp.type, typing.TypeFloat32): if operator := OPERATOR_MAP.get(inp.operator, None): yield wasm.Statement(f'f32.{operator}') return - if isinstance(inp.type, ourlang.OurTypeFloat64): + if isinstance(inp.type, typing.TypeFloat64): if operator := OPERATOR_MAP.get(inp.operator, None): yield wasm.Statement(f'f64.{operator}') return @@ -115,18 +116,18 @@ def expression(inp: ourlang.Expression) -> Statements: if isinstance(inp, ourlang.UnaryOp): yield from expression(inp.right) - if isinstance(inp.type, ourlang.OurTypeFloat32): + if isinstance(inp.type, typing.TypeFloat32): if inp.operator in ourlang.WEBASSEMBLY_BUILDIN_FLOAT_OPS: yield wasm.Statement(f'f32.{inp.operator}') return - if isinstance(inp.type, ourlang.OurTypeFloat64): + if isinstance(inp.type, typing.TypeFloat64): if inp.operator in ourlang.WEBASSEMBLY_BUILDIN_FLOAT_OPS: yield wasm.Statement(f'f64.{inp.operator}') return - if isinstance(inp.type, ourlang.OurTypeInt32): + if isinstance(inp.type, typing.TypeInt32): if inp.operator == 'len': - if isinstance(inp.right.type, ourlang.OurTypeBytes): + if isinstance(inp.right.type, typing.TypeBytes): yield wasm.Statement('i32.load') return @@ -140,7 +141,7 @@ def expression(inp: ourlang.Expression) -> Statements: return if isinstance(inp, ourlang.AccessBytesIndex): - if not isinstance(inp.type, ourlang.OurTypeUInt8): + if not isinstance(inp.type, typing.TypeUInt8): raise NotImplementedError(inp, inp.type) yield from expression(inp.varref) @@ -149,14 +150,14 @@ def expression(inp: ourlang.Expression) -> Statements: return if isinstance(inp, ourlang.AccessStructMember): - if isinstance(inp.member.type, ourlang.OurTypeUInt8): + if isinstance(inp.member.type, typing.TypeUInt8): mtyp = 'i32' else: # FIXME: Properly implement this # inp.type.render() is also a hack that doesn't really work consistently if not isinstance(inp.member.type, ( - ourlang.OurTypeInt32, ourlang.OurTypeFloat32, - ourlang.OurTypeInt64, ourlang.OurTypeFloat64, + typing.TypeInt32, typing.TypeFloat32, + typing.TypeInt64, typing.TypeFloat64, )): raise NotImplementedError mtyp = inp.member.type.render() @@ -169,8 +170,8 @@ def expression(inp: ourlang.Expression) -> Statements: # FIXME: Properly implement this # inp.type.render() is also a hack that doesn't really work consistently if not isinstance(inp.type, ( - ourlang.OurTypeInt32, ourlang.OurTypeFloat32, - ourlang.OurTypeInt64, ourlang.OurTypeFloat64, + typing.TypeInt32, typing.TypeFloat32, + typing.TypeInt64, typing.TypeFloat64, )): raise NotImplementedError(inp, inp.type) @@ -213,7 +214,7 @@ def statement(inp: ourlang.Statement) -> Statements: raise NotImplementedError(statement, inp) -def function_argument(inp: Tuple[str, ourlang.OurType]) -> wasm.Param: +def function_argument(inp: Tuple[str, typing.TypeBase]) -> wasm.Param: return (inp[0], type_(inp[1]), ) def import_(inp: ourlang.Function) -> wasm.Import: @@ -352,8 +353,8 @@ def _generate_tuple_constructor(inp: ourlang.TupleConstructor) -> Statements: # FIXME: Properly implement this # inp.type.render() is also a hack that doesn't really work consistently if not isinstance(member.type, ( - ourlang.OurTypeInt32, ourlang.OurTypeFloat32, - ourlang.OurTypeInt64, ourlang.OurTypeFloat64, + typing.TypeInt32, typing.TypeFloat32, + typing.TypeInt64, typing.TypeFloat64, )): raise NotImplementedError @@ -370,14 +371,14 @@ def _generate_struct_constructor(inp: ourlang.StructConstructor) -> Statements: yield wasm.Statement('local.set', '$___new_reference___addr') for member in inp.struct.members: - if isinstance(member.type, ourlang.OurTypeUInt8): + if isinstance(member.type, typing.TypeUInt8): mtyp = 'i32' else: # FIXME: Properly implement this # inp.type.render() is also a hack that doesn't really work consistently if not isinstance(member.type, ( - ourlang.OurTypeInt32, ourlang.OurTypeFloat32, - ourlang.OurTypeInt64, ourlang.OurTypeFloat64, + typing.TypeInt32, typing.TypeFloat32, + typing.TypeInt64, typing.TypeFloat64, )): raise NotImplementedError mtyp = member.type.render() diff --git a/phasm/ourlang.py b/phasm/ourlang.py index 99156be..64bdedc 100644 --- a/phasm/ourlang.py +++ b/phasm/ourlang.py @@ -9,138 +9,17 @@ from typing_extensions import Final WEBASSEMBLY_BUILDIN_FLOAT_OPS: Final = ('abs', 'sqrt', 'ceil', 'floor', 'trunc', 'nearest', ) -class OurType: - """ - Type base class - """ - __slots__ = () - - def render(self) -> str: - """ - Renders the type back to source code format - - This'll look like Python code. - """ - raise NotImplementedError(self, 'render') - - def alloc_size(self) -> int: - """ - When allocating this type in memory, how many bytes do we need to reserve? - """ - raise NotImplementedError(self, 'alloc_size') - -class OurTypeNone(OurType): - """ - The None (or Void) type - """ - __slots__ = () - - def render(self) -> str: - return 'None' - -class OurTypeUInt8(OurType): - """ - The Integer type, unsigned and 8 bits wide - """ - __slots__ = () - - def render(self) -> str: - return 'u8' - - def alloc_size(self) -> int: - return 4 # Int32 under the hood - -class OurTypeInt32(OurType): - """ - The Integer type, signed and 32 bits wide - """ - __slots__ = () - - def render(self) -> str: - return 'i32' - - def alloc_size(self) -> int: - return 4 - -class OurTypeInt64(OurType): - """ - The Integer type, signed and 64 bits wide - """ - __slots__ = () - - def render(self) -> str: - return 'i64' - - def alloc_size(self) -> int: - return 8 - -class OurTypeFloat32(OurType): - """ - The Float type, 32 bits wide - """ - __slots__ = () - - def render(self) -> str: - return 'f32' - - def alloc_size(self) -> int: - return 4 - -class OurTypeFloat64(OurType): - """ - The Float type, 64 bits wide - """ - __slots__ = () - - def render(self) -> str: - return 'f64' - - def alloc_size(self) -> int: - return 8 - -class OurTypeBytes(OurType): - """ - The bytes type - """ - __slots__ = () - - def render(self) -> str: - return 'bytes' - -class TupleMember: - """ - Represents a tuple member - """ - def __init__(self, idx: int, type_: OurType, offset: int) -> None: - self.idx = idx - self.type = type_ - self.offset = offset - -class OurTypeTuple(OurType): - """ - The tuple type - """ - __slots__ = ('members', ) - - members: List[TupleMember] - - def __init__(self) -> None: - self.members = [] - - def render(self) -> str: - mems = ', '.join(x.type.render() for x in self.members) - return f'({mems}, )' - - def render_internal_name(self) -> str: - mems = '@'.join(x.type.render() for x in self.members) - assert ' ' not in mems, 'Not implement yet: subtuples' - return f'tuple@{mems}' - - def alloc_size(self) -> int: - return sum( - x.type.alloc_size() - for x in self.members - ) +from .typing import ( + TypeBase, + TypeNone, + TypeBool, + TypeUInt8, + TypeInt32, TypeInt64, + TypeFloat32, TypeFloat64, + TypeBytes, + TypeTuple, TypeTupleMember, + TypeStruct, TypeStructMember, +) class Expression: """ @@ -148,9 +27,9 @@ class Expression: """ __slots__ = ('type', ) - type: OurType + type: TypeBase - def __init__(self, type_: OurType) -> None: + def __init__(self, type_: TypeBase) -> None: self.type = type_ def render(self) -> str: @@ -175,7 +54,7 @@ class ConstantUInt8(Constant): value: int - def __init__(self, type_: OurTypeUInt8, value: int) -> None: + def __init__(self, type_: TypeUInt8, value: int) -> None: super().__init__(type_) self.value = value @@ -190,7 +69,7 @@ class ConstantInt32(Constant): value: int - def __init__(self, type_: OurTypeInt32, value: int) -> None: + def __init__(self, type_: TypeInt32, value: int) -> None: super().__init__(type_) self.value = value @@ -205,7 +84,7 @@ class ConstantInt64(Constant): value: int - def __init__(self, type_: OurTypeInt64, value: int) -> None: + def __init__(self, type_: TypeInt64, value: int) -> None: super().__init__(type_) self.value = value @@ -220,7 +99,7 @@ class ConstantFloat32(Constant): value: float - def __init__(self, type_: OurTypeFloat32, value: float) -> None: + def __init__(self, type_: TypeFloat32, value: float) -> None: super().__init__(type_) self.value = value @@ -235,7 +114,7 @@ class ConstantFloat64(Constant): value: float - def __init__(self, type_: OurTypeFloat64, value: float) -> None: + def __init__(self, type_: TypeFloat64, value: float) -> None: super().__init__(type_) self.value = value @@ -250,7 +129,7 @@ class VariableReference(Expression): name: str - def __init__(self, type_: OurType, name: str) -> None: + def __init__(self, type_: TypeBase, name: str) -> None: super().__init__(type_) self.name = name @@ -267,7 +146,7 @@ class BinaryOp(Expression): left: Expression right: Expression - def __init__(self, type_: OurType, operator: str, left: Expression, right: Expression) -> None: + def __init__(self, type_: TypeBase, operator: str, left: Expression, right: Expression) -> None: super().__init__(type_) self.operator = operator @@ -286,7 +165,7 @@ class UnaryOp(Expression): operator: str right: Expression - def __init__(self, type_: OurType, operator: str, right: Expression) -> None: + def __init__(self, type_: TypeBase, operator: str, right: Expression) -> None: super().__init__(type_) self.operator = operator @@ -336,7 +215,7 @@ class AccessBytesIndex(Expression): varref: VariableReference index: Expression - def __init__(self, type_: OurType, varref: VariableReference, index: Expression) -> None: + def __init__(self, type_: TypeBase, varref: VariableReference, index: Expression) -> None: super().__init__(type_) self.varref = varref @@ -352,9 +231,9 @@ class AccessStructMember(Expression): __slots__ = ('varref', 'member', ) varref: VariableReference - member: 'StructMember' + member: TypeStructMember - def __init__(self, varref: VariableReference, member: 'StructMember') -> None: + def __init__(self, varref: VariableReference, member: TypeStructMember) -> None: super().__init__(member.type) self.varref = varref @@ -370,9 +249,9 @@ class AccessTupleMember(Expression): __slots__ = ('varref', 'member', ) varref: VariableReference - member: TupleMember + member: TypeTupleMember - def __init__(self, varref: VariableReference, member: TupleMember, ) -> None: + def __init__(self, varref: VariableReference, member: TypeTupleMember, ) -> None: super().__init__(member.type) self.varref = varref @@ -465,8 +344,8 @@ class Function: exported: bool imported: bool statements: List[Statement] - returns: OurType - posonlyargs: List[Tuple[str, OurType]] + returns: TypeBase + posonlyargs: List[Tuple[str, TypeBase]] def __init__(self, name: str, lineno: int) -> None: self.name = name @@ -474,7 +353,7 @@ class Function: self.exported = False self.imported = False self.statements = [] - self.returns = OurTypeNone() + self.returns = TypeNone() self.posonlyargs = [] def render(self) -> str: @@ -509,9 +388,9 @@ class StructConstructor(Function): """ __slots__ = ('struct', ) - struct: 'Struct' + struct: TypeStruct - def __init__(self, struct: 'Struct') -> None: + def __init__(self, struct: TypeStruct) -> None: super().__init__(f'@{struct.name}@__init___@', -1) self.returns = struct @@ -527,9 +406,9 @@ class TupleConstructor(Function): """ __slots__ = ('tuple', ) - tuple: OurTypeTuple + tuple: TypeTuple - def __init__(self, tuple_: OurTypeTuple) -> None: + def __init__(self, tuple_: TypeTuple) -> None: name = tuple_.render_internal_name() super().__init__(f'@{name}@__init___@', -1) @@ -541,85 +420,25 @@ class TupleConstructor(Function): self.tuple = tuple_ -class StructMember: - """ - Represents a struct member - """ - def __init__(self, name: str, type_: OurType, offset: int) -> None: - self.name = name - self.type = type_ - self.offset = offset - -class Struct(OurType): - """ - A struct has named properties - """ - __slots__ = ('name', 'lineno', 'members', ) - - name: str - lineno: int - members: List[StructMember] - - def __init__(self, name: str, lineno: int) -> None: - self.name = name - self.lineno = lineno - self.members = [] - - def get_member(self, name: str) -> Optional[StructMember]: - """ - Returns a member by name - """ - for mem in self.members: - if mem.name == name: - return mem - - return None - - def render(self) -> str: - """ - Renders the type back to source code format - - This'll look like Python code. - """ - return self.name - - def render_definition(self) -> str: - """ - Renders the definition back to source code format - - This'll look like Python code. - """ - result = f'class {self.name}:\n' - for mem in self.members: - result += f' {mem.name}: {mem.type.render()}\n' - - return result - - def alloc_size(self) -> int: - return sum( - x.type.alloc_size() - for x in self.members - ) - class Module: """ A module is a file and consists of functions """ __slots__ = ('types', 'functions', 'structs', ) - types: Dict[str, OurType] + types: Dict[str, TypeBase] functions: Dict[str, Function] - structs: Dict[str, Struct] + structs: Dict[str, TypeStruct] def __init__(self) -> None: self.types = { - 'None': OurTypeNone(), - 'u8': OurTypeUInt8(), - 'i32': OurTypeInt32(), - 'i64': OurTypeInt64(), - 'f32': OurTypeFloat32(), - 'f64': OurTypeFloat64(), - 'bytes': OurTypeBytes(), + 'None': TypeNone(), + 'u8': TypeUInt8(), + 'i32': TypeInt32(), + 'i64': TypeInt64(), + 'f32': TypeFloat32(), + 'f64': TypeFloat64(), + 'bytes': TypeBytes(), } self.functions = {} self.structs = {} @@ -653,7 +472,7 @@ class StaticError(Exception): An error found during static analysis """ -OurLocals = Dict[str, OurType] +OurLocals = Dict[str, TypeBase] class OurVisitor: """ @@ -683,7 +502,7 @@ class OurVisitor: module.functions[res.name] = res - if isinstance(res, Struct): + if isinstance(res, TypeStruct): if res.name in module.structs: raise StaticError( f'{res.name} already defined on line {module.structs[res.name].lineno}' @@ -700,7 +519,7 @@ class OurVisitor: return module - def pre_visit_Module_stmt(self, module: Module, node: ast.stmt) -> Union[Function, Struct]: + def pre_visit_Module_stmt(self, module: Module, node: ast.stmt) -> Union[Function, TypeStruct]: if isinstance(node, ast.FunctionDef): return self.pre_visit_Module_FunctionDef(module, node) @@ -750,8 +569,8 @@ class OurVisitor: return function - def pre_visit_Module_ClassDef(self, module: Module, node: ast.ClassDef) -> Struct: - struct = Struct(node.name, node.lineno) + def pre_visit_Module_ClassDef(self, module: Module, node: ast.ClassDef) -> TypeStruct: + struct = TypeStruct(node.name, node.lineno) _not_implemented(not node.bases, 'ClassDef.bases') _not_implemented(not node.keywords, 'ClassDef.keywords') @@ -772,7 +591,7 @@ class OurVisitor: if stmt.simple != 1: raise NotImplementedError('Class with non-simple arguments') - member = StructMember(stmt.target.id, self.visit_type(module, stmt.annotation), offset) + member = TypeStructMember(stmt.target.id, self.visit_type(module, stmt.annotation), offset) struct.members.append(member) offset += member.type.alloc_size() @@ -831,7 +650,7 @@ class OurVisitor: raise NotImplementedError(f'{node} as stmt in FunctionDef') - def visit_Module_FunctionDef_expr(self, module: Module, function: Function, our_locals: OurLocals, exp_type: OurType, node: ast.expr) -> Expression: + def visit_Module_FunctionDef_expr(self, module: Module, function: Function, our_locals: OurLocals, exp_type: TypeBase, node: ast.expr) -> Expression: if isinstance(node, ast.BinOp): if isinstance(node.op, ast.Add): operator = '+' @@ -924,7 +743,7 @@ class OurVisitor: if not isinstance(node.ctx, ast.Load): _raise_static_error(node, 'Must be load context') - if not isinstance(exp_type, OurTypeTuple): + if not isinstance(exp_type, TypeTuple): _raise_static_error(node, f'Expression is expecting a {exp_type.render()}, not a tuple') if len(exp_type.members) != len(node.elts): @@ -943,7 +762,7 @@ class OurVisitor: raise NotImplementedError(f'{node} as expr in FunctionDef') - def visit_Module_FunctionDef_Call(self, module: Module, function: Function, our_locals: OurLocals, exp_type: OurType, node: ast.Call) -> Union[FunctionCall, UnaryOp]: + def visit_Module_FunctionDef_Call(self, module: Module, function: Function, our_locals: OurLocals, exp_type: TypeBase, node: ast.Call) -> Union[FunctionCall, UnaryOp]: if node.keywords: _raise_static_error(node, 'Keyword calling not supported') # Yet? @@ -958,7 +777,7 @@ class OurVisitor: func = module.functions[struct_constructor.name] elif node.func.id in WEBASSEMBLY_BUILDIN_FLOAT_OPS: - if not isinstance(exp_type, (OurTypeFloat32, OurTypeFloat64, )): + if not isinstance(exp_type, (TypeFloat32, TypeFloat64, )): _raise_static_error(node, f'Cannot make {node.func.id} result in {exp_type}') if 1 != len(node.args): @@ -970,7 +789,7 @@ class OurVisitor: self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.args[0]), ) elif node.func.id == 'len': - if not isinstance(exp_type, OurTypeInt32): + if not isinstance(exp_type, TypeInt32): _raise_static_error(node, f'Cannot make {node.func.id} result in {exp_type}') if 1 != len(node.args): @@ -1000,7 +819,7 @@ class OurVisitor: ) return result - def visit_Module_FunctionDef_Attribute(self, module: Module, function: Function, our_locals: OurLocals, exp_type: OurType, node: ast.Attribute) -> Expression: + def visit_Module_FunctionDef_Attribute(self, module: Module, function: Function, our_locals: OurLocals, exp_type: TypeBase, node: ast.Attribute) -> Expression: if not isinstance(node.value, ast.Name): _raise_static_error(node, 'Must reference a name') @@ -1011,7 +830,7 @@ class OurVisitor: _raise_static_error(node, f'Undefined variable {node.value.id}') node_typ = our_locals[node.value.id] - if not isinstance(node_typ, Struct): + if not isinstance(node_typ, TypeStruct): _raise_static_error(node, f'Cannot take attribute of non-struct {node.value.id}') member = node_typ.get_member(node.attr) @@ -1026,7 +845,7 @@ class OurVisitor: member, ) - def visit_Module_FunctionDef_Subscript(self, module: Module, function: Function, our_locals: OurLocals, exp_type: OurType, node: ast.Subscript) -> Expression: + def visit_Module_FunctionDef_Subscript(self, module: Module, function: Function, our_locals: OurLocals, exp_type: TypeBase, node: ast.Subscript) -> Expression: if not isinstance(node.value, ast.Name): _raise_static_error(node, 'Must reference a name') @@ -1045,14 +864,14 @@ class OurVisitor: module, function, our_locals, module.types['i32'], node.slice.value, ) - if isinstance(node_typ, OurTypeBytes): + if isinstance(node_typ, TypeBytes): return AccessBytesIndex( module.types['u8'], VariableReference(node_typ, node.value.id), slice_expr, ) - if isinstance(node_typ, OurTypeTuple): + if isinstance(node_typ, TypeTuple): if not isinstance(slice_expr, ConstantInt32): _raise_static_error(node, 'Must subscript using a constant index') @@ -1072,13 +891,13 @@ class OurVisitor: _raise_static_error(node, f'Cannot take index of {node_typ.render()} {node.value.id}') - def visit_Module_FunctionDef_Constant(self, module: Module, function: Function, exp_type: OurType, node: ast.Constant) -> Expression: + def visit_Module_FunctionDef_Constant(self, module: Module, function: Function, exp_type: TypeBase, node: ast.Constant) -> Expression: del module del function _not_implemented(node.kind is None, 'Constant.kind') - if isinstance(exp_type, OurTypeUInt8): + if isinstance(exp_type, TypeUInt8): if not isinstance(node.value, int): _raise_static_error(node, 'Expected integer value') @@ -1086,7 +905,7 @@ class OurVisitor: return ConstantUInt8(exp_type, node.value) - if isinstance(exp_type, OurTypeInt32): + if isinstance(exp_type, TypeInt32): if not isinstance(node.value, int): _raise_static_error(node, 'Expected integer value') @@ -1094,7 +913,7 @@ class OurVisitor: return ConstantInt32(exp_type, node.value) - if isinstance(exp_type, OurTypeInt64): + if isinstance(exp_type, TypeInt64): if not isinstance(node.value, int): _raise_static_error(node, 'Expected integer value') @@ -1102,7 +921,7 @@ class OurVisitor: return ConstantInt64(exp_type, node.value) - if isinstance(exp_type, OurTypeFloat32): + if isinstance(exp_type, TypeFloat32): if not isinstance(node.value, (float, int, )): _raise_static_error(node, 'Expected float value') @@ -1110,7 +929,7 @@ class OurVisitor: return ConstantFloat32(exp_type, node.value) - if isinstance(exp_type, OurTypeFloat64): + if isinstance(exp_type, TypeFloat64): if not isinstance(node.value, (float, int, )): _raise_static_error(node, 'Expected float value') @@ -1120,7 +939,7 @@ class OurVisitor: raise NotImplementedError(f'{node} as const for type {exp_type.render()}') - def visit_type(self, module: Module, node: ast.expr) -> OurType: + def visit_type(self, module: Module, node: ast.expr) -> TypeBase: if isinstance(node, ast.Constant): if node.value is None: return module.types['None'] @@ -1143,12 +962,12 @@ class OurVisitor: if not isinstance(node.ctx, ast.Load): _raise_static_error(node, 'Must be load context') - result = OurTypeTuple() + result = TypeTuple() offset = 0 for idx, elt in enumerate(node.elts): - member = TupleMember(idx, self.visit_type(module, elt), offset) + member = TypeTupleMember(idx, self.visit_type(module, elt), offset) result.members.append(member) offset += member.type.alloc_size() diff --git a/phasm/typing.py b/phasm/typing.py new file mode 100644 index 0000000..4252fdb --- /dev/null +++ b/phasm/typing.py @@ -0,0 +1,206 @@ +""" +The phasm type system +""" +from typing import Optional, List + +class TypeBase: + """ + TypeBase base class + """ + __slots__ = () + + def render(self) -> str: + """ + Renders the type back to source code format + + This'll look like Python code. + """ + raise NotImplementedError(self, 'render') + + def alloc_size(self) -> int: + """ + When allocating this type in memory, how many bytes do we need to reserve? + """ + raise NotImplementedError(self, 'alloc_size') + +class TypeNone(TypeBase): + """ + The None (or Void) type + """ + __slots__ = () + + def render(self) -> str: + return 'None' + +class TypeBool(TypeBase): + """ + The boolean type + """ + __slots__ = () + + def render(self) -> str: + return 'bool' + +class TypeUInt8(TypeBase): + """ + The Integer type, unsigned and 8 bits wide + """ + __slots__ = () + + def render(self) -> str: + return 'u8' + + def alloc_size(self) -> int: + return 4 # Int32 under the hood + +class TypeInt32(TypeBase): + """ + The Integer type, signed and 32 bits wide + """ + __slots__ = () + + def render(self) -> str: + return 'i32' + + def alloc_size(self) -> int: + return 4 + +class TypeInt64(TypeBase): + """ + The Integer type, signed and 64 bits wide + """ + __slots__ = () + + def render(self) -> str: + return 'i64' + + def alloc_size(self) -> int: + return 8 + +class TypeFloat32(TypeBase): + """ + The Float type, 32 bits wide + """ + __slots__ = () + + def render(self) -> str: + return 'f32' + + def alloc_size(self) -> int: + return 4 + +class TypeFloat64(TypeBase): + """ + The Float type, 64 bits wide + """ + __slots__ = () + + def render(self) -> str: + return 'f64' + + def alloc_size(self) -> int: + return 8 + +class TypeBytes(TypeBase): + """ + The bytes type + """ + __slots__ = () + + def render(self) -> str: + return 'bytes' + +class TypeTupleMember: + """ + Represents a tuple member + """ + def __init__(self, idx: int, type_: TypeBase, offset: int) -> None: + self.idx = idx + self.type = type_ + self.offset = offset + +class TypeTuple(TypeBase): + """ + The tuple type + """ + __slots__ = ('members', ) + + members: List[TypeTupleMember] + + def __init__(self) -> None: + self.members = [] + + def render(self) -> str: + mems = ', '.join(x.type.render() for x in self.members) + return f'({mems}, )' + + def render_internal_name(self) -> str: + mems = '@'.join(x.type.render() for x in self.members) + assert ' ' not in mems, 'Not implement yet: subtuples' + return f'tuple@{mems}' + + def alloc_size(self) -> int: + return sum( + x.type.alloc_size() + for x in self.members + ) + +class TypeStructMember: + """ + Represents a struct member + """ + def __init__(self, name: str, type_: TypeBase, offset: int) -> None: + self.name = name + self.type = type_ + self.offset = offset + +class TypeStruct(TypeBase): + """ + A struct has named properties + """ + __slots__ = ('name', 'lineno', 'members', ) + + name: str + lineno: int + members: List[TypeStructMember] + + def __init__(self, name: str, lineno: int) -> None: + self.name = name + self.lineno = lineno + self.members = [] + + def get_member(self, name: str) -> Optional[TypeStructMember]: + """ + Returns a member by name + """ + for mem in self.members: + if mem.name == name: + return mem + + return None + + def render(self) -> str: + """ + Renders the type back to source code format + + This'll look like Python code. + """ + return self.name + + def render_definition(self) -> str: + """ + Renders the definition back to source code format + + This'll look like Python code. + """ + result = f'class {self.name}:\n' + for mem in self.members: + result += f' {mem.name}: {mem.type.render()}\n' + + return result + + def alloc_size(self) -> int: + return sum( + x.type.alloc_size() + for x in self.members + ) From 89ad648f348d842d53135cbb8d94254752e86cb1 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 9 Jul 2022 14:04:40 +0200 Subject: [PATCH 47/73] Moved rendering to codestyle, parsing to parser Also, removed name argument when parsing, wasn't used. --- phasm/__main__.py | 8 +- phasm/codestyle.py | 195 ++++++ phasm/compiler.py | 109 ++-- phasm/exceptions.py | 8 + phasm/ourlang.py | 718 +--------------------- phasm/parser.py | 577 +++++++++++++++++ phasm/typing.py | 61 +- phasm/utils.py | 16 - phasm/wasm.py | 3 - pylintrc | 2 +- tests/integration/helpers.py | 16 +- tests/integration/test_fib.py | 2 +- tests/integration/test_runtime_checks.py | 2 +- tests/integration/test_simple.py | 78 ++- tests/integration/test_static_checking.py | 12 +- 15 files changed, 951 insertions(+), 856 deletions(-) create mode 100644 phasm/codestyle.py create mode 100644 phasm/exceptions.py create mode 100644 phasm/parser.py delete mode 100644 phasm/utils.py diff --git a/phasm/__main__.py b/phasm/__main__.py index 7c65724..bc94cdb 100644 --- a/phasm/__main__.py +++ b/phasm/__main__.py @@ -4,8 +4,8 @@ Functions for using this module from CLI import sys -from .utils import our_process -from .compiler import module +from .parser import phasm_parse +from .compiler import phasm_compile def main(source: str, sink: str) -> int: """ @@ -15,8 +15,8 @@ def main(source: str, sink: str) -> int: with open(source, 'r') as fil: code_py = fil.read() - our_module = our_process(code_py, source) - wasm_module = module(our_module) + our_module = phasm_parse(code_py) + wasm_module = phasm_compile(our_module) code_wat = wasm_module.to_wat() with open(sink, 'w') as fil: diff --git a/phasm/codestyle.py b/phasm/codestyle.py new file mode 100644 index 0000000..5d217f5 --- /dev/null +++ b/phasm/codestyle.py @@ -0,0 +1,195 @@ +""" +This module generates source code based on the parsed AST + +It's intented to be a "any color, as long as it's black" kind of renderer +""" +from typing import Generator + +from . import ourlang +from . import typing + +def phasm_render(inp: ourlang.Module) -> str: + """ + Public method for rendering a Phasm module into Phasm code + """ + return module(inp) + +Statements = Generator[str, None, None] + +def type_(inp: typing.TypeBase) -> str: + """ + Render: Type (name) + """ + if isinstance(inp, typing.TypeNone): + return 'None' + + if isinstance(inp, typing.TypeBool): + return 'bool' + + if isinstance(inp, typing.TypeUInt8): + return 'u8' + + if isinstance(inp, typing.TypeInt32): + return 'i32' + + if isinstance(inp, typing.TypeInt64): + return 'i64' + + if isinstance(inp, typing.TypeFloat32): + return 'f32' + + if isinstance(inp, typing.TypeFloat64): + return 'f64' + + if isinstance(inp, typing.TypeBytes): + return 'bytes' + + if isinstance(inp, typing.TypeTuple): + mems = ', '.join( + type_(x.type) + for x in inp.members + ) + + return f'({mems}, )' + + if isinstance(inp, typing.TypeStruct): + return inp.name + + raise NotImplementedError(type_, inp) + +def struct_definition(inp: typing.TypeStruct) -> str: + """ + Render: TypeStruct's definition + """ + result = f'class {inp.name}:\n' + for mem in inp.members: + result += f' {mem.name}: {type_(mem.type)}\n' + + return result + +def expression(inp: ourlang.Expression) -> str: + """ + Render: A Phasm expression + """ + if isinstance(inp, (ourlang.ConstantUInt8, ourlang.ConstantInt32, ourlang.ConstantInt64, )): + return str(inp.value) + + if isinstance(inp, (ourlang.ConstantFloat32, ourlang.ConstantFloat64, )): + # These might not round trip if the original constant + # could not fit in the given float type + return str(inp.value) + + if isinstance(inp, ourlang.VariableReference): + return str(inp.name) + + if isinstance(inp, ourlang.UnaryOp): + if ( + inp.operator in ourlang.WEBASSEMBLY_BUILDIN_FLOAT_OPS + or inp.operator in ourlang.WEBASSEMBLY_BUILDIN_BYTES_OPS): + return f'{inp.operator}({expression(inp.right)})' + + return f'{inp.operator}{expression(inp.right)}' + + if isinstance(inp, ourlang.BinaryOp): + return f'{expression(inp.left)} {inp.operator} {expression(inp.right)}' + + if isinstance(inp, ourlang.FunctionCall): + args = ', '.join( + expression(arg) + for arg in inp.arguments + ) + + if isinstance(inp.function, ourlang.StructConstructor): + return f'{inp.function.struct.name}({args})' + + if isinstance(inp.function, ourlang.TupleConstructor): + return f'({args}, )' + + return f'{inp.function.name}({args})' + + if isinstance(inp, ourlang.AccessBytesIndex): + return f'{expression(inp.varref)}[{expression(inp.index)}]' + + if isinstance(inp, ourlang.AccessStructMember): + return f'{expression(inp.varref)}.{inp.member.name}' + + if isinstance(inp, ourlang.AccessTupleMember): + return f'{expression(inp.varref)}[{inp.member.idx}]' + + raise NotImplementedError(expression, inp) + +def statement(inp: ourlang.Statement) -> Statements: + """ + Render: A list of Phasm statements + """ + if isinstance(inp, ourlang.StatementPass): + yield 'pass' + return + + if isinstance(inp, ourlang.StatementReturn): + yield f'return {expression(inp.value)}' + return + + if isinstance(inp, ourlang.StatementIf): + yield f'if {expression(inp.test)}:' + + for stmt in inp.statements: + for line in statement(stmt): + yield f' {line}' if line else '' + + yield '' + return + + raise NotImplementedError(statement, inp) + +def function(inp: ourlang.Function) -> str: + """ + Render: Function body + + Imported functions only have "pass" as a body. Later on we might replace + this by the function documentation, if any. + """ + result = '' + if inp.exported: + result += '@exported\n' + if inp.imported: + result += '@imported\n' + + args = ', '.join( + f'{x}: {type_(y)}' + for x, y in inp.posonlyargs + ) + + result += f'def {inp.name}({args}) -> {type_(inp.returns)}:\n' + + if inp.imported: + result += ' pass\n' + else: + for stmt in inp.statements: + for line in statement(stmt): + result += f' {line}\n' if line else '\n' + + return result + + +def module(inp: ourlang.Module) -> str: + """ + Render: Module + """ + result = '' + + for struct in inp.structs.values(): + if result: + result += '\n' + result += struct_definition(struct) + + for func in inp.functions.values(): + if func.lineno < 0: + # Buildin (-2) or auto generated (-1) + continue + + if result: + result += '\n' + result += function(func) + + return result diff --git a/phasm/compiler.py b/phasm/compiler.py index 979543e..4ad4bd2 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -1,7 +1,7 @@ """ This module contains the code to convert parsed Ourlang into WebAssembly code """ -from typing import Generator, Tuple +from typing import Generator from . import ourlang from . import typing @@ -9,7 +9,28 @@ from . import wasm Statements = Generator[wasm.Statement, None, None] +LOAD_STORE_TYPE_MAP = { + typing.TypeUInt8: 'i32', + typing.TypeInt32: 'i32', + typing.TypeInt64: 'i64', + typing.TypeFloat32: 'f32', + typing.TypeFloat64: 'f64', +} +""" +When generating code, we sometimes need to load or store simple values +""" + +def phasm_compile(inp: ourlang.Module) -> wasm.Module: + """ + Public method for compiling a parsed Phasm module into + a WebAssembly module + """ + return module(inp) + def type_(inp: typing.TypeBase) -> wasm.WasmType: + """ + Compile: type + """ if isinstance(inp, typing.TypeNone): return wasm.WasmTypeNone() @@ -60,6 +81,9 @@ I64_OPERATOR_MAP = { # TODO: Introduce UInt32 type } def expression(inp: ourlang.Expression) -> Statements: + """ + Compile: Any expression + """ if isinstance(inp, ourlang.ConstantUInt8): yield wasm.Statement('i32.const', str(inp.value)) return @@ -150,42 +174,40 @@ def expression(inp: ourlang.Expression) -> Statements: return if isinstance(inp, ourlang.AccessStructMember): - if isinstance(inp.member.type, typing.TypeUInt8): - mtyp = 'i32' - else: - # FIXME: Properly implement this - # inp.type.render() is also a hack that doesn't really work consistently - if not isinstance(inp.member.type, ( - typing.TypeInt32, typing.TypeFloat32, - typing.TypeInt64, typing.TypeFloat64, - )): - raise NotImplementedError - mtyp = inp.member.type.render() + mtyp = LOAD_STORE_TYPE_MAP.get(inp.member.type.__class__) + if mtyp is None: + # In the future might extend this by having structs or tuples + # as members of struct or tuples + raise NotImplementedError(expression, inp, inp.member) yield from expression(inp.varref) yield wasm.Statement(f'{mtyp}.load', 'offset=' + str(inp.member.offset)) return if isinstance(inp, ourlang.AccessTupleMember): - # FIXME: Properly implement this - # inp.type.render() is also a hack that doesn't really work consistently - if not isinstance(inp.type, ( - typing.TypeInt32, typing.TypeFloat32, - typing.TypeInt64, typing.TypeFloat64, - )): - raise NotImplementedError(inp, inp.type) + mtyp = LOAD_STORE_TYPE_MAP.get(inp.member.type.__class__) + if mtyp is None: + # In the future might extend this by having structs or tuples + # as members of struct or tuples + raise NotImplementedError(expression, inp, inp.member) yield from expression(inp.varref) - yield wasm.Statement(inp.type.render() + '.load', 'offset=' + str(inp.member.offset)) + yield wasm.Statement(f'{mtyp}.load', 'offset=' + str(inp.member.offset)) return raise NotImplementedError(expression, inp) def statement_return(inp: ourlang.StatementReturn) -> Statements: + """ + Compile: Return statement + """ yield from expression(inp.value) yield wasm.Statement('return') def statement_if(inp: ourlang.StatementIf) -> Statements: + """ + Compile: If statement + """ yield from expression(inp.test) yield wasm.Statement('if') @@ -201,6 +223,9 @@ def statement_if(inp: ourlang.StatementIf) -> Statements: yield wasm.Statement('end') def statement(inp: ourlang.Statement) -> Statements: + """ + Compile: any statement + """ if isinstance(inp, ourlang.StatementReturn): yield from statement_return(inp) return @@ -214,10 +239,16 @@ def statement(inp: ourlang.Statement) -> Statements: raise NotImplementedError(statement, inp) -def function_argument(inp: Tuple[str, typing.TypeBase]) -> wasm.Param: +def function_argument(inp: ourlang.FunctionParam) -> wasm.Param: + """ + Compile: function argument + """ return (inp[0], type_(inp[1]), ) def import_(inp: ourlang.Function) -> wasm.Import: + """ + Compile: imported function + """ assert inp.imported return wasm.Import( @@ -232,6 +263,9 @@ def import_(inp: ourlang.Function) -> wasm.Import: ) def function(inp: ourlang.Function) -> wasm.Function: + """ + Compile: function + """ assert not inp.imported if isinstance(inp, ourlang.TupleConstructor): @@ -269,6 +303,9 @@ def function(inp: ourlang.Function) -> wasm.Function: ) def module(inp: ourlang.Module) -> wasm.Module: + """ + Compile: module + """ result = wasm.Module() result.imports = [ @@ -350,17 +387,15 @@ def _generate_tuple_constructor(inp: ourlang.TupleConstructor) -> Statements: yield wasm.Statement('local.set', '$___new_reference___addr') for member in inp.tuple.members: - # FIXME: Properly implement this - # inp.type.render() is also a hack that doesn't really work consistently - if not isinstance(member.type, ( - typing.TypeInt32, typing.TypeFloat32, - typing.TypeInt64, typing.TypeFloat64, - )): - raise NotImplementedError + mtyp = LOAD_STORE_TYPE_MAP.get(member.type.__class__) + if mtyp is None: + # In the future might extend this by having structs or tuples + # as members of struct or tuples + raise NotImplementedError(expression, inp, member) yield wasm.Statement('local.get', '$___new_reference___addr') yield wasm.Statement('local.get', f'$arg{member.idx}') - yield wasm.Statement(f'{member.type.render()}.store', 'offset=' + str(member.offset)) + yield wasm.Statement(f'{mtyp}.store', 'offset=' + str(member.offset)) yield wasm.Statement('local.get', '$___new_reference___addr') @@ -371,17 +406,11 @@ def _generate_struct_constructor(inp: ourlang.StructConstructor) -> Statements: yield wasm.Statement('local.set', '$___new_reference___addr') for member in inp.struct.members: - if isinstance(member.type, typing.TypeUInt8): - mtyp = 'i32' - else: - # FIXME: Properly implement this - # inp.type.render() is also a hack that doesn't really work consistently - if not isinstance(member.type, ( - typing.TypeInt32, typing.TypeFloat32, - typing.TypeInt64, typing.TypeFloat64, - )): - raise NotImplementedError - mtyp = member.type.render() + mtyp = LOAD_STORE_TYPE_MAP.get(member.type.__class__) + if mtyp is None: + # In the future might extend this by having structs or tuples + # as members of struct or tuples + raise NotImplementedError(expression, inp, member) yield wasm.Statement('local.get', '$___new_reference___addr') yield wasm.Statement('local.get', f'${member.name}') diff --git a/phasm/exceptions.py b/phasm/exceptions.py new file mode 100644 index 0000000..b459c22 --- /dev/null +++ b/phasm/exceptions.py @@ -0,0 +1,8 @@ +""" +Exceptions for the phasm compiler +""" + +class StaticError(Exception): + """ + An error found during static analysis + """ diff --git a/phasm/ourlang.py b/phasm/ourlang.py index 64bdedc..1903328 100644 --- a/phasm/ourlang.py +++ b/phasm/ourlang.py @@ -1,13 +1,12 @@ """ Contains the syntax tree for ourlang """ -from typing import Any, Dict, List, Optional, NoReturn, Union, Tuple - -import ast +from typing import Dict, List, Tuple from typing_extensions import Final WEBASSEMBLY_BUILDIN_FLOAT_OPS: Final = ('abs', 'sqrt', 'ceil', 'floor', 'trunc', 'nearest', ) +WEBASSEMBLY_BUILDIN_BYTES_OPS: Final = ('len', ) from .typing import ( TypeBase, @@ -32,14 +31,6 @@ class Expression: def __init__(self, type_: TypeBase) -> None: self.type = type_ - def render(self) -> str: - """ - Renders the expression back to source code format - - This'll look like Python code. - """ - raise NotImplementedError(self, 'render') - class Constant(Expression): """ An constant value expression within a statement @@ -58,9 +49,6 @@ class ConstantUInt8(Constant): super().__init__(type_) self.value = value - def render(self) -> str: - return str(self.value) - class ConstantInt32(Constant): """ An Int32 constant value expression within a statement @@ -73,9 +61,6 @@ class ConstantInt32(Constant): super().__init__(type_) self.value = value - def render(self) -> str: - return str(self.value) - class ConstantInt64(Constant): """ An Int64 constant value expression within a statement @@ -88,9 +73,6 @@ class ConstantInt64(Constant): super().__init__(type_) self.value = value - def render(self) -> str: - return str(self.value) - class ConstantFloat32(Constant): """ An Float32 constant value expression within a statement @@ -103,9 +85,6 @@ class ConstantFloat32(Constant): super().__init__(type_) self.value = value - def render(self) -> str: - return str(self.value) - class ConstantFloat64(Constant): """ An Float64 constant value expression within a statement @@ -118,9 +97,6 @@ class ConstantFloat64(Constant): super().__init__(type_) self.value = value - def render(self) -> str: - return str(self.value) - class VariableReference(Expression): """ An variable reference expression within a statement @@ -133,8 +109,20 @@ class VariableReference(Expression): super().__init__(type_) self.name = name - def render(self) -> str: - return str(self.name) +class UnaryOp(Expression): + """ + A unary operator expression within a statement + """ + __slots__ = ('operator', 'right', ) + + operator: str + right: Expression + + def __init__(self, type_: TypeBase, operator: str, right: Expression) -> None: + super().__init__(type_) + + self.operator = operator + self.right = right class BinaryOp(Expression): """ @@ -153,30 +141,6 @@ class BinaryOp(Expression): self.left = left self.right = right - def render(self) -> str: - return f'{self.left.render()} {self.operator} {self.right.render()}' - -class UnaryOp(Expression): - """ - A unary operator expression within a statement - """ - __slots__ = ('operator', 'right', ) - - operator: str - right: Expression - - def __init__(self, type_: TypeBase, operator: str, right: Expression) -> None: - super().__init__(type_) - - self.operator = operator - self.right = right - - def render(self) -> str: - if self.operator in WEBASSEMBLY_BUILDIN_FLOAT_OPS or self.operator == 'len': - return f'{self.operator}({self.right.render()})' - - return f'{self.operator}{self.right.render()}' - class FunctionCall(Expression): """ A function call expression within a statement @@ -192,20 +156,6 @@ class FunctionCall(Expression): self.function = function self.arguments = [] - def render(self) -> str: - args = ', '.join( - arg.render() - for arg in self.arguments - ) - - if isinstance(self.function, StructConstructor): - return f'{self.function.struct.name}({args})' - - if isinstance(self.function, TupleConstructor): - return f'({args}, )' - - return f'{self.function.name}({args})' - class AccessBytesIndex(Expression): """ Access a bytes index for reading @@ -221,9 +171,6 @@ class AccessBytesIndex(Expression): self.varref = varref self.index = index - def render(self) -> str: - return f'{self.varref.render()}[{self.index.render()}]' - class AccessStructMember(Expression): """ Access a struct member for reading of writing @@ -239,9 +186,6 @@ class AccessStructMember(Expression): self.varref = varref self.member = member - def render(self) -> str: - return f'{self.varref.render()}.{self.member.name}' - class AccessTupleMember(Expression): """ Access a tuple member for reading of writing @@ -257,22 +201,17 @@ class AccessTupleMember(Expression): self.varref = varref self.member = member - def render(self) -> str: - return f'{self.varref.render()}[{self.member.idx}]' - class Statement: """ A statement within a function """ __slots__ = () - def render(self) -> List[str]: - """ - Renders the type back to source code format - - This'll look like Python code. - """ - raise NotImplementedError(self, 'render') +class StatementPass(Statement): + """ + A pass statement + """ + __slots__ = () class StatementReturn(Statement): """ @@ -283,14 +222,6 @@ class StatementReturn(Statement): def __init__(self, value: Expression) -> None: self.value = value - def render(self) -> List[str]: - """ - Renders the type back to source code format - - This'll look like Python code. - """ - return [f'return {self.value.render()}'] - class StatementIf(Statement): """ An if statement within a function @@ -306,32 +237,7 @@ class StatementIf(Statement): self.statements = [] self.else_statements = [] - def render(self) -> List[str]: - """ - Renders the type back to source code format - - This'll look like Python code. - """ - result = [f'if {self.test.render()}:'] - - for stmt in self.statements: - result.extend( - f' {line}' if line else '' - for line in stmt.render() - ) - - result.append('') - - return result - -class StatementPass(Statement): - """ - A pass statement - """ - __slots__ = () - - def render(self) -> List[str]: - return ['pass'] +FunctionParam = Tuple[str, TypeBase] class Function: """ @@ -345,7 +251,7 @@ class Function: imported: bool statements: List[Statement] returns: TypeBase - posonlyargs: List[Tuple[str, TypeBase]] + posonlyargs: List[FunctionParam] def __init__(self, name: str, lineno: int) -> None: self.name = name @@ -356,35 +262,12 @@ class Function: self.returns = TypeNone() self.posonlyargs = [] - def render(self) -> str: - """ - Renders the function back to source code format - - This'll look like Python code. - """ - statements = self.statements - - result = '' - if self.exported: - result += '@exported\n' - if self.imported: - result += '@imported\n' - statements = [StatementPass()] - - args = ', '.join( - f'{x}: {y.render()}' - for x, y in self.posonlyargs - ) - - result += f'def {self.name}({args}) -> {self.returns.render()}:\n' - for stmt in statements: - for line in stmt.render(): - result += f' {line}\n' if line else '\n' - return result - class StructConstructor(Function): """ The constructor method for a struct + + A function will generated to instantiate a struct. The arguments + will be the defaults """ __slots__ = ('struct', ) @@ -442,552 +325,3 @@ class Module: } self.functions = {} self.structs = {} - - def render(self) -> str: - """ - Renders the module back to source code format - - This'll look like Python code. - """ - result = '' - - for struct in self.structs.values(): - if result: - result += '\n' - result += struct.render_definition() - - for function in self.functions.values(): - if function.lineno < 0: - # Buildin (-2) or auto generated (-1) - continue - - if result: - result += '\n' - result += function.render() - - return result - -class StaticError(Exception): - """ - An error found during static analysis - """ - -OurLocals = Dict[str, TypeBase] - -class OurVisitor: - """ - Class to visit a Python syntax tree and create an ourlang syntax tree - """ - - # pylint: disable=C0103,C0116,C0301,R0201,R0912 - - def __init__(self) -> None: - pass - - def visit_Module(self, node: ast.Module) -> Module: - module = Module() - - _not_implemented(not node.type_ignores, 'Module.type_ignores') - - # Second pass for the types - - for stmt in node.body: - res = self.pre_visit_Module_stmt(module, stmt) - - if isinstance(res, Function): - if res.name in module.functions: - raise StaticError( - f'{res.name} already defined on line {module.functions[res.name].lineno}' - ) - - module.functions[res.name] = res - - if isinstance(res, TypeStruct): - if res.name in module.structs: - raise StaticError( - f'{res.name} already defined on line {module.structs[res.name].lineno}' - ) - - module.structs[res.name] = res - constructor = StructConstructor(res) - module.functions[constructor.name] = constructor - - # Second pass for the function bodies - - for stmt in node.body: - self.visit_Module_stmt(module, stmt) - - return module - - def pre_visit_Module_stmt(self, module: Module, node: ast.stmt) -> Union[Function, TypeStruct]: - if isinstance(node, ast.FunctionDef): - return self.pre_visit_Module_FunctionDef(module, node) - - if isinstance(node, ast.ClassDef): - return self.pre_visit_Module_ClassDef(module, node) - - raise NotImplementedError(f'{node} on Module') - - def pre_visit_Module_FunctionDef(self, module: Module, node: ast.FunctionDef) -> Function: - function = Function(node.name, node.lineno) - - _not_implemented(not node.args.posonlyargs, 'FunctionDef.args.posonlyargs') - - for arg in node.args.args: - if not arg.annotation: - _raise_static_error(node, 'Type is required') - - function.posonlyargs.append(( - arg.arg, - self.visit_type(module, arg.annotation), - )) - - _not_implemented(not node.args.vararg, 'FunctionDef.args.vararg') - _not_implemented(not node.args.kwonlyargs, 'FunctionDef.args.kwonlyargs') - _not_implemented(not node.args.kw_defaults, 'FunctionDef.args.kw_defaults') - _not_implemented(not node.args.kwarg, 'FunctionDef.args.kwarg') - _not_implemented(not node.args.defaults, 'FunctionDef.args.defaults') - - # Do stmts at the end so we have the return value - - for decorator in node.decorator_list: - if not isinstance(decorator, ast.Name): - _raise_static_error(decorator, 'Function decorators must be string') - if not isinstance(decorator.ctx, ast.Load): - _raise_static_error(decorator, 'Must be load context') - _not_implemented(decorator.id in ('exported', 'imported'), 'Custom decorators') - - if decorator.id == 'exported': - function.exported = True - else: - function.imported = True - - if node.returns: - function.returns = self.visit_type(module, node.returns) - - _not_implemented(not node.type_comment, 'FunctionDef.type_comment') - - return function - - def pre_visit_Module_ClassDef(self, module: Module, node: ast.ClassDef) -> TypeStruct: - struct = TypeStruct(node.name, node.lineno) - - _not_implemented(not node.bases, 'ClassDef.bases') - _not_implemented(not node.keywords, 'ClassDef.keywords') - _not_implemented(not node.decorator_list, 'ClassDef.decorator_list') - - offset = 0 - - for stmt in node.body: - if not isinstance(stmt, ast.AnnAssign): - raise NotImplementedError(f'Class with {stmt} nodes') - - if not isinstance(stmt.target, ast.Name): - raise NotImplementedError('Class with default values') - - if not stmt.value is None: - raise NotImplementedError('Class with default values') - - if stmt.simple != 1: - raise NotImplementedError('Class with non-simple arguments') - - member = TypeStructMember(stmt.target.id, self.visit_type(module, stmt.annotation), offset) - - struct.members.append(member) - offset += member.type.alloc_size() - - return struct - - def visit_Module_stmt(self, module: Module, node: ast.stmt) -> None: - if isinstance(node, ast.FunctionDef): - self.visit_Module_FunctionDef(module, node) - return - - if isinstance(node, ast.ClassDef): - return - - raise NotImplementedError(f'{node} on Module') - - def visit_Module_FunctionDef(self, module: Module, node: ast.FunctionDef) -> None: - function = module.functions[node.name] - - our_locals = dict(function.posonlyargs) - - for stmt in node.body: - function.statements.append( - self.visit_Module_FunctionDef_stmt(module, function, our_locals, stmt) - ) - - def visit_Module_FunctionDef_stmt(self, module: Module, function: Function, our_locals: OurLocals, node: ast.stmt) -> Statement: - if isinstance(node, ast.Return): - if node.value is None: - # TODO: Implement methods without return values - _raise_static_error(node, 'Return must have an argument') - - return StatementReturn( - self.visit_Module_FunctionDef_expr(module, function, our_locals, function.returns, node.value) - ) - - if isinstance(node, ast.If): - result = StatementIf( - self.visit_Module_FunctionDef_expr(module, function, our_locals, function.returns, node.test) - ) - - for stmt in node.body: - result.statements.append( - self.visit_Module_FunctionDef_stmt(module, function, our_locals, stmt) - ) - - for stmt in node.orelse: - result.else_statements.append( - self.visit_Module_FunctionDef_stmt(module, function, our_locals, stmt) - ) - - return result - - if isinstance(node, ast.Pass): - return StatementPass() - - raise NotImplementedError(f'{node} as stmt in FunctionDef') - - def visit_Module_FunctionDef_expr(self, module: Module, function: Function, our_locals: OurLocals, exp_type: TypeBase, node: ast.expr) -> Expression: - if isinstance(node, ast.BinOp): - if isinstance(node.op, ast.Add): - operator = '+' - elif isinstance(node.op, ast.Sub): - operator = '-' - elif isinstance(node.op, ast.Mult): - operator = '*' - else: - raise NotImplementedError(f'Operator {node.op}') - - # Assume the type doesn't change when descending into a binary operator - # e.g. you can do `"hello" * 3` with the code below (yet) - - return BinaryOp( - exp_type, - operator, - self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.left), - self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.right), - ) - - if isinstance(node, ast.UnaryOp): - if isinstance(node.op, ast.UAdd): - operator = '+' - elif isinstance(node.op, ast.USub): - operator = '-' - else: - raise NotImplementedError(f'Operator {node.op}') - - return UnaryOp( - exp_type, - operator, - self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.operand), - ) - - if isinstance(node, ast.Compare): - if 1 < len(node.ops): - raise NotImplementedError('Multiple operators') - - if isinstance(node.ops[0], ast.Gt): - operator = '>' - elif isinstance(node.ops[0], ast.Eq): - operator = '==' - elif isinstance(node.ops[0], ast.Lt): - operator = '<' - else: - raise NotImplementedError(f'Operator {node.ops}') - - # Assume the type doesn't change when descending into a binary operator - # e.g. you can do `"hello" * 3` with the code below (yet) - - return BinaryOp( - exp_type, - operator, - self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.left), - self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.comparators[0]), - ) - - if isinstance(node, ast.Call): - return self.visit_Module_FunctionDef_Call(module, function, our_locals, exp_type, node) - - if isinstance(node, ast.Constant): - return self.visit_Module_FunctionDef_Constant( - module, function, exp_type, node, - ) - - if isinstance(node, ast.Attribute): - return self.visit_Module_FunctionDef_Attribute( - module, function, our_locals, exp_type, node, - ) - - if isinstance(node, ast.Subscript): - return self.visit_Module_FunctionDef_Subscript( - module, function, our_locals, exp_type, node, - ) - - if isinstance(node, ast.Name): - if not isinstance(node.ctx, ast.Load): - _raise_static_error(node, 'Must be load context') - - if node.id not in our_locals: - _raise_static_error(node, 'Undefined variable') - - act_type = our_locals[node.id] - if exp_type != act_type: - _raise_static_error(node, f'Expected {exp_type.render()}, {node.id} is actually {act_type.render()}') - - return VariableReference(act_type, node.id) - - if isinstance(node, ast.Tuple): - if not isinstance(node.ctx, ast.Load): - _raise_static_error(node, 'Must be load context') - - if not isinstance(exp_type, TypeTuple): - _raise_static_error(node, f'Expression is expecting a {exp_type.render()}, not a tuple') - - if len(exp_type.members) != len(node.elts): - _raise_static_error(node, f'Expression is expecting a tuple of size {len(exp_type.members)}, but {len(node.elts)} are given') - - tuple_constructor = TupleConstructor(exp_type) - - func = module.functions[tuple_constructor.name] - - result = FunctionCall(func) - result.arguments = [ - self.visit_Module_FunctionDef_expr(module, function, our_locals, mem.type, arg_node) - for arg_node, mem in zip(node.elts, exp_type.members) - ] - return result - - raise NotImplementedError(f'{node} as expr in FunctionDef') - - def visit_Module_FunctionDef_Call(self, module: Module, function: Function, our_locals: OurLocals, exp_type: TypeBase, node: ast.Call) -> Union[FunctionCall, UnaryOp]: - if node.keywords: - _raise_static_error(node, 'Keyword calling not supported') # Yet? - - if not isinstance(node.func, ast.Name): - raise NotImplementedError(f'Calling methods that are not a name {node.func}') - if not isinstance(node.func.ctx, ast.Load): - _raise_static_error(node, 'Must be load context') - - if node.func.id in module.structs: - struct = module.structs[node.func.id] - struct_constructor = StructConstructor(struct) - - func = module.functions[struct_constructor.name] - elif node.func.id in WEBASSEMBLY_BUILDIN_FLOAT_OPS: - if not isinstance(exp_type, (TypeFloat32, TypeFloat64, )): - _raise_static_error(node, f'Cannot make {node.func.id} result in {exp_type}') - - if 1 != len(node.args): - _raise_static_error(node, f'Function {node.func.id} requires 1 arguments but {len(node.args)} are given') - - return UnaryOp( - exp_type, - 'sqrt', - self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.args[0]), - ) - elif node.func.id == 'len': - if not isinstance(exp_type, TypeInt32): - _raise_static_error(node, f'Cannot make {node.func.id} result in {exp_type}') - - if 1 != len(node.args): - _raise_static_error(node, f'Function {node.func.id} requires 1 arguments but {len(node.args)} are given') - - return UnaryOp( - exp_type, - 'len', - self.visit_Module_FunctionDef_expr(module, function, our_locals, module.types['bytes'], node.args[0]), - ) - else: - if node.func.id not in module.functions: - _raise_static_error(node, 'Call to undefined function') - - func = module.functions[node.func.id] - - if func.returns != exp_type: - _raise_static_error(node, f'Expected {exp_type.render()}, {func.name} actually returns {func.returns.render()}') - - if len(func.posonlyargs) != len(node.args): - _raise_static_error(node, f'Function {node.func.id} requires {len(func.posonlyargs)} arguments but {len(node.args)} are given') - - result = FunctionCall(func) - result.arguments.extend( - self.visit_Module_FunctionDef_expr(module, function, our_locals, arg_type, arg_expr) - for arg_expr, (_, arg_type) in zip(node.args, func.posonlyargs) - ) - return result - - def visit_Module_FunctionDef_Attribute(self, module: Module, function: Function, our_locals: OurLocals, exp_type: TypeBase, node: ast.Attribute) -> Expression: - if not isinstance(node.value, ast.Name): - _raise_static_error(node, 'Must reference a name') - - if not isinstance(node.ctx, ast.Load): - _raise_static_error(node, 'Must be load context') - - if not node.value.id in our_locals: - _raise_static_error(node, f'Undefined variable {node.value.id}') - - node_typ = our_locals[node.value.id] - if not isinstance(node_typ, TypeStruct): - _raise_static_error(node, f'Cannot take attribute of non-struct {node.value.id}') - - member = node_typ.get_member(node.attr) - if member is None: - _raise_static_error(node, f'{node_typ.name} has no attribute {node.attr}') - - if exp_type != member.type: - _raise_static_error(node, f'Expected {exp_type.render()}, {node.value.id}.{member.name} is actually {member.type.render()}') - - return AccessStructMember( - VariableReference(node_typ, node.value.id), - member, - ) - - def visit_Module_FunctionDef_Subscript(self, module: Module, function: Function, our_locals: OurLocals, exp_type: TypeBase, node: ast.Subscript) -> Expression: - if not isinstance(node.value, ast.Name): - _raise_static_error(node, 'Must reference a name') - - if not isinstance(node.slice, ast.Index): - _raise_static_error(node, 'Must subscript using an index') - - if not isinstance(node.ctx, ast.Load): - _raise_static_error(node, 'Must be load context') - - if not node.value.id in our_locals: - _raise_static_error(node, f'Undefined variable {node.value.id}') - - node_typ = our_locals[node.value.id] - - slice_expr = self.visit_Module_FunctionDef_expr( - module, function, our_locals, module.types['i32'], node.slice.value, - ) - - if isinstance(node_typ, TypeBytes): - return AccessBytesIndex( - module.types['u8'], - VariableReference(node_typ, node.value.id), - slice_expr, - ) - - if isinstance(node_typ, TypeTuple): - if not isinstance(slice_expr, ConstantInt32): - _raise_static_error(node, 'Must subscript using a constant index') - - idx = slice_expr.value - - if len(node_typ.members) <= idx: - _raise_static_error(node, f'Index {idx} out of bounds for tuple {node.value.id}') - - member = node_typ.members[idx] - if exp_type != member.type: - _raise_static_error(node, f'Expected {exp_type.render()}, {node.value.id}[{idx}] is actually {member.type.render()}') - - return AccessTupleMember( - VariableReference(node_typ, node.value.id), - member, - ) - - _raise_static_error(node, f'Cannot take index of {node_typ.render()} {node.value.id}') - - def visit_Module_FunctionDef_Constant(self, module: Module, function: Function, exp_type: TypeBase, node: ast.Constant) -> Expression: - del module - del function - - _not_implemented(node.kind is None, 'Constant.kind') - - if isinstance(exp_type, TypeUInt8): - if not isinstance(node.value, int): - _raise_static_error(node, 'Expected integer value') - - # FIXME: Range check - - return ConstantUInt8(exp_type, node.value) - - if isinstance(exp_type, TypeInt32): - if not isinstance(node.value, int): - _raise_static_error(node, 'Expected integer value') - - # FIXME: Range check - - return ConstantInt32(exp_type, node.value) - - if isinstance(exp_type, TypeInt64): - if not isinstance(node.value, int): - _raise_static_error(node, 'Expected integer value') - - # FIXME: Range check - - return ConstantInt64(exp_type, node.value) - - if isinstance(exp_type, TypeFloat32): - if not isinstance(node.value, (float, int, )): - _raise_static_error(node, 'Expected float value') - - # FIXME: Range check - - return ConstantFloat32(exp_type, node.value) - - if isinstance(exp_type, TypeFloat64): - if not isinstance(node.value, (float, int, )): - _raise_static_error(node, 'Expected float value') - - # FIXME: Range check - - return ConstantFloat64(exp_type, node.value) - - raise NotImplementedError(f'{node} as const for type {exp_type.render()}') - - def visit_type(self, module: Module, node: ast.expr) -> TypeBase: - if isinstance(node, ast.Constant): - if node.value is None: - return module.types['None'] - - _raise_static_error(node, f'Unrecognized type {node.value}') - - if isinstance(node, ast.Name): - if not isinstance(node.ctx, ast.Load): - _raise_static_error(node, 'Must be load context') - - if node.id in module.types: - return module.types[node.id] - - if node.id in module.structs: - return module.structs[node.id] - - _raise_static_error(node, f'Unrecognized type {node.id}') - - if isinstance(node, ast.Tuple): - if not isinstance(node.ctx, ast.Load): - _raise_static_error(node, 'Must be load context') - - result = TypeTuple() - - offset = 0 - - for idx, elt in enumerate(node.elts): - member = TypeTupleMember(idx, self.visit_type(module, elt), offset) - - result.members.append(member) - offset += member.type.alloc_size() - - key = result.render_internal_name() - - if key not in module.types: - module.types[key] = result - constructor = TupleConstructor(result) - module.functions[constructor.name] = constructor - - return module.types[key] - - raise NotImplementedError(f'{node} as type') - -def _not_implemented(check: Any, msg: str) -> None: - if not check: - raise NotImplementedError(msg) - -def _raise_static_error(node: Union[ast.mod, ast.stmt, ast.expr], msg: str) -> NoReturn: - raise StaticError( - f'Static error on line {node.lineno}: {msg}' - ) diff --git a/phasm/parser.py b/phasm/parser.py new file mode 100644 index 0000000..dda06e6 --- /dev/null +++ b/phasm/parser.py @@ -0,0 +1,577 @@ +""" +Parses the source code from the plain text into a syntax tree +""" +from typing import Any, Dict, NoReturn, Union + +import ast + +from .typing import ( + TypeBase, + TypeUInt8, + TypeInt32, + TypeInt64, + TypeFloat32, + TypeFloat64, + TypeBytes, + TypeStruct, + TypeStructMember, + TypeTuple, + TypeTupleMember, +) + +from . import codestyle +from .exceptions import StaticError +from .ourlang import ( + WEBASSEMBLY_BUILDIN_FLOAT_OPS, + + Module, + Function, + + Expression, + AccessBytesIndex, AccessStructMember, AccessTupleMember, + BinaryOp, + ConstantFloat32, ConstantFloat64, ConstantInt32, ConstantInt64, ConstantUInt8, + FunctionCall, + StructConstructor, TupleConstructor, + UnaryOp, VariableReference, + + Statement, + StatementIf, StatementPass, StatementReturn, +) + +def phasm_parse(source: str) -> Module: + """ + Public method for parsing Phasm code into a Phasm Module + """ + res = ast.parse(source, '') + + our_visitor = OurVisitor() + return our_visitor.visit_Module(res) + +OurLocals = Dict[str, TypeBase] + +class OurVisitor: + """ + Class to visit a Python syntax tree and create an ourlang syntax tree + + We're (ab)using the Python AST parser to give us a leg up + + At some point, we may deviate from Python syntax. If nothing else, + we probably won't keep up with the Python syntax changes. + """ + + # pylint: disable=C0103,C0116,C0301,R0201,R0912 + + def __init__(self) -> None: + pass + + def visit_Module(self, node: ast.Module) -> Module: + module = Module() + + _not_implemented(not node.type_ignores, 'Module.type_ignores') + + # Second pass for the types + + for stmt in node.body: + res = self.pre_visit_Module_stmt(module, stmt) + + if isinstance(res, Function): + if res.name in module.functions: + raise StaticError( + f'{res.name} already defined on line {module.functions[res.name].lineno}' + ) + + module.functions[res.name] = res + + if isinstance(res, TypeStruct): + if res.name in module.structs: + raise StaticError( + f'{res.name} already defined on line {module.structs[res.name].lineno}' + ) + + module.structs[res.name] = res + constructor = StructConstructor(res) + module.functions[constructor.name] = constructor + + # Second pass for the function bodies + + for stmt in node.body: + self.visit_Module_stmt(module, stmt) + + return module + + def pre_visit_Module_stmt(self, module: Module, node: ast.stmt) -> Union[Function, TypeStruct]: + if isinstance(node, ast.FunctionDef): + return self.pre_visit_Module_FunctionDef(module, node) + + if isinstance(node, ast.ClassDef): + return self.pre_visit_Module_ClassDef(module, node) + + raise NotImplementedError(f'{node} on Module') + + def pre_visit_Module_FunctionDef(self, module: Module, node: ast.FunctionDef) -> Function: + function = Function(node.name, node.lineno) + + _not_implemented(not node.args.posonlyargs, 'FunctionDef.args.posonlyargs') + + for arg in node.args.args: + if not arg.annotation: + _raise_static_error(node, 'Type is required') + + function.posonlyargs.append(( + arg.arg, + self.visit_type(module, arg.annotation), + )) + + _not_implemented(not node.args.vararg, 'FunctionDef.args.vararg') + _not_implemented(not node.args.kwonlyargs, 'FunctionDef.args.kwonlyargs') + _not_implemented(not node.args.kw_defaults, 'FunctionDef.args.kw_defaults') + _not_implemented(not node.args.kwarg, 'FunctionDef.args.kwarg') + _not_implemented(not node.args.defaults, 'FunctionDef.args.defaults') + + # Do stmts at the end so we have the return value + + for decorator in node.decorator_list: + if not isinstance(decorator, ast.Name): + _raise_static_error(decorator, 'Function decorators must be string') + if not isinstance(decorator.ctx, ast.Load): + _raise_static_error(decorator, 'Must be load context') + _not_implemented(decorator.id in ('exported', 'imported'), 'Custom decorators') + + if decorator.id == 'exported': + function.exported = True + else: + function.imported = True + + if node.returns: + function.returns = self.visit_type(module, node.returns) + + _not_implemented(not node.type_comment, 'FunctionDef.type_comment') + + return function + + def pre_visit_Module_ClassDef(self, module: Module, node: ast.ClassDef) -> TypeStruct: + struct = TypeStruct(node.name, node.lineno) + + _not_implemented(not node.bases, 'ClassDef.bases') + _not_implemented(not node.keywords, 'ClassDef.keywords') + _not_implemented(not node.decorator_list, 'ClassDef.decorator_list') + + offset = 0 + + for stmt in node.body: + if not isinstance(stmt, ast.AnnAssign): + raise NotImplementedError(f'Class with {stmt} nodes') + + if not isinstance(stmt.target, ast.Name): + raise NotImplementedError('Class with default values') + + if not stmt.value is None: + raise NotImplementedError('Class with default values') + + if stmt.simple != 1: + raise NotImplementedError('Class with non-simple arguments') + + member = TypeStructMember(stmt.target.id, self.visit_type(module, stmt.annotation), offset) + + struct.members.append(member) + offset += member.type.alloc_size() + + return struct + + def visit_Module_stmt(self, module: Module, node: ast.stmt) -> None: + if isinstance(node, ast.FunctionDef): + self.visit_Module_FunctionDef(module, node) + return + + if isinstance(node, ast.ClassDef): + return + + raise NotImplementedError(f'{node} on Module') + + def visit_Module_FunctionDef(self, module: Module, node: ast.FunctionDef) -> None: + function = module.functions[node.name] + + our_locals = dict(function.posonlyargs) + + for stmt in node.body: + function.statements.append( + self.visit_Module_FunctionDef_stmt(module, function, our_locals, stmt) + ) + + def visit_Module_FunctionDef_stmt(self, module: Module, function: Function, our_locals: OurLocals, node: ast.stmt) -> Statement: + if isinstance(node, ast.Return): + if node.value is None: + # TODO: Implement methods without return values + _raise_static_error(node, 'Return must have an argument') + + return StatementReturn( + self.visit_Module_FunctionDef_expr(module, function, our_locals, function.returns, node.value) + ) + + if isinstance(node, ast.If): + result = StatementIf( + self.visit_Module_FunctionDef_expr(module, function, our_locals, function.returns, node.test) + ) + + for stmt in node.body: + result.statements.append( + self.visit_Module_FunctionDef_stmt(module, function, our_locals, stmt) + ) + + for stmt in node.orelse: + result.else_statements.append( + self.visit_Module_FunctionDef_stmt(module, function, our_locals, stmt) + ) + + return result + + if isinstance(node, ast.Pass): + return StatementPass() + + raise NotImplementedError(f'{node} as stmt in FunctionDef') + + def visit_Module_FunctionDef_expr(self, module: Module, function: Function, our_locals: OurLocals, exp_type: TypeBase, node: ast.expr) -> Expression: + if isinstance(node, ast.BinOp): + if isinstance(node.op, ast.Add): + operator = '+' + elif isinstance(node.op, ast.Sub): + operator = '-' + elif isinstance(node.op, ast.Mult): + operator = '*' + else: + raise NotImplementedError(f'Operator {node.op}') + + # Assume the type doesn't change when descending into a binary operator + # e.g. you can do `"hello" * 3` with the code below (yet) + + return BinaryOp( + exp_type, + operator, + self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.left), + self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.right), + ) + + if isinstance(node, ast.UnaryOp): + if isinstance(node.op, ast.UAdd): + operator = '+' + elif isinstance(node.op, ast.USub): + operator = '-' + else: + raise NotImplementedError(f'Operator {node.op}') + + return UnaryOp( + exp_type, + operator, + self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.operand), + ) + + if isinstance(node, ast.Compare): + if 1 < len(node.ops): + raise NotImplementedError('Multiple operators') + + if isinstance(node.ops[0], ast.Gt): + operator = '>' + elif isinstance(node.ops[0], ast.Eq): + operator = '==' + elif isinstance(node.ops[0], ast.Lt): + operator = '<' + else: + raise NotImplementedError(f'Operator {node.ops}') + + # Assume the type doesn't change when descending into a binary operator + # e.g. you can do `"hello" * 3` with the code below (yet) + + return BinaryOp( + exp_type, + operator, + self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.left), + self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.comparators[0]), + ) + + if isinstance(node, ast.Call): + return self.visit_Module_FunctionDef_Call(module, function, our_locals, exp_type, node) + + if isinstance(node, ast.Constant): + return self.visit_Module_FunctionDef_Constant( + module, function, exp_type, node, + ) + + if isinstance(node, ast.Attribute): + return self.visit_Module_FunctionDef_Attribute( + module, function, our_locals, exp_type, node, + ) + + if isinstance(node, ast.Subscript): + return self.visit_Module_FunctionDef_Subscript( + module, function, our_locals, exp_type, node, + ) + + if isinstance(node, ast.Name): + if not isinstance(node.ctx, ast.Load): + _raise_static_error(node, 'Must be load context') + + if node.id not in our_locals: + _raise_static_error(node, 'Undefined variable') + + act_type = our_locals[node.id] + if exp_type != act_type: + _raise_static_error(node, f'Expected {codestyle.type_(exp_type)}, {node.id} is actually {codestyle.type_(act_type)}') + + return VariableReference(act_type, node.id) + + if isinstance(node, ast.Tuple): + if not isinstance(node.ctx, ast.Load): + _raise_static_error(node, 'Must be load context') + + if not isinstance(exp_type, TypeTuple): + _raise_static_error(node, f'Expression is expecting a {codestyle.type_(exp_type)}, not a tuple') + + if len(exp_type.members) != len(node.elts): + _raise_static_error(node, f'Expression is expecting a tuple of size {len(exp_type.members)}, but {len(node.elts)} are given') + + tuple_constructor = TupleConstructor(exp_type) + + func = module.functions[tuple_constructor.name] + + result = FunctionCall(func) + result.arguments = [ + self.visit_Module_FunctionDef_expr(module, function, our_locals, mem.type, arg_node) + for arg_node, mem in zip(node.elts, exp_type.members) + ] + return result + + raise NotImplementedError(f'{node} as expr in FunctionDef') + + def visit_Module_FunctionDef_Call(self, module: Module, function: Function, our_locals: OurLocals, exp_type: TypeBase, node: ast.Call) -> Union[FunctionCall, UnaryOp]: + if node.keywords: + _raise_static_error(node, 'Keyword calling not supported') # Yet? + + if not isinstance(node.func, ast.Name): + raise NotImplementedError(f'Calling methods that are not a name {node.func}') + if not isinstance(node.func.ctx, ast.Load): + _raise_static_error(node, 'Must be load context') + + if node.func.id in module.structs: + struct = module.structs[node.func.id] + struct_constructor = StructConstructor(struct) + + func = module.functions[struct_constructor.name] + elif node.func.id in WEBASSEMBLY_BUILDIN_FLOAT_OPS: + if not isinstance(exp_type, (TypeFloat32, TypeFloat64, )): + _raise_static_error(node, f'Cannot make {node.func.id} result in {codestyle.type_(exp_type)}') + + if 1 != len(node.args): + _raise_static_error(node, f'Function {node.func.id} requires 1 arguments but {len(node.args)} are given') + + return UnaryOp( + exp_type, + 'sqrt', + self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.args[0]), + ) + elif node.func.id == 'len': + if not isinstance(exp_type, TypeInt32): + _raise_static_error(node, f'Cannot make {node.func.id} result in {exp_type}') + + if 1 != len(node.args): + _raise_static_error(node, f'Function {node.func.id} requires 1 arguments but {len(node.args)} are given') + + return UnaryOp( + exp_type, + 'len', + self.visit_Module_FunctionDef_expr(module, function, our_locals, module.types['bytes'], node.args[0]), + ) + else: + if node.func.id not in module.functions: + _raise_static_error(node, 'Call to undefined function') + + func = module.functions[node.func.id] + + if func.returns != exp_type: + _raise_static_error(node, f'Expected {codestyle.type_(exp_type)}, {func.name} actually returns {codestyle.type_(func.returns)}') + + if len(func.posonlyargs) != len(node.args): + _raise_static_error(node, f'Function {node.func.id} requires {len(func.posonlyargs)} arguments but {len(node.args)} are given') + + result = FunctionCall(func) + result.arguments.extend( + self.visit_Module_FunctionDef_expr(module, function, our_locals, arg_type, arg_expr) + for arg_expr, (_, arg_type) in zip(node.args, func.posonlyargs) + ) + return result + + def visit_Module_FunctionDef_Attribute(self, module: Module, function: Function, our_locals: OurLocals, exp_type: TypeBase, node: ast.Attribute) -> Expression: + del module + del function + + if not isinstance(node.value, ast.Name): + _raise_static_error(node, 'Must reference a name') + + if not isinstance(node.ctx, ast.Load): + _raise_static_error(node, 'Must be load context') + + if not node.value.id in our_locals: + _raise_static_error(node, f'Undefined variable {node.value.id}') + + node_typ = our_locals[node.value.id] + if not isinstance(node_typ, TypeStruct): + _raise_static_error(node, f'Cannot take attribute of non-struct {node.value.id}') + + member = node_typ.get_member(node.attr) + if member is None: + _raise_static_error(node, f'{node_typ.name} has no attribute {node.attr}') + + if exp_type != member.type: + _raise_static_error(node, f'Expected {codestyle.type_(exp_type)}, {node.value.id}.{member.name} is actually {codestyle.type_(member.type)}') + + return AccessStructMember( + VariableReference(node_typ, node.value.id), + member, + ) + + def visit_Module_FunctionDef_Subscript(self, module: Module, function: Function, our_locals: OurLocals, exp_type: TypeBase, node: ast.Subscript) -> Expression: + if not isinstance(node.value, ast.Name): + _raise_static_error(node, 'Must reference a name') + + if not isinstance(node.slice, ast.Index): + _raise_static_error(node, 'Must subscript using an index') + + if not isinstance(node.ctx, ast.Load): + _raise_static_error(node, 'Must be load context') + + if not node.value.id in our_locals: + _raise_static_error(node, f'Undefined variable {node.value.id}') + + node_typ = our_locals[node.value.id] + + slice_expr = self.visit_Module_FunctionDef_expr( + module, function, our_locals, module.types['i32'], node.slice.value, + ) + + if isinstance(node_typ, TypeBytes): + return AccessBytesIndex( + module.types['u8'], + VariableReference(node_typ, node.value.id), + slice_expr, + ) + + if isinstance(node_typ, TypeTuple): + if not isinstance(slice_expr, ConstantInt32): + _raise_static_error(node, 'Must subscript using a constant index') + + idx = slice_expr.value + + if len(node_typ.members) <= idx: + _raise_static_error(node, f'Index {idx} out of bounds for tuple {node.value.id}') + + member = node_typ.members[idx] + if exp_type != member.type: + _raise_static_error(node, f'Expected {codestyle.type_(exp_type)}, {node.value.id}[{idx}] is actually {codestyle.type_(member.type)}') + + return AccessTupleMember( + VariableReference(node_typ, node.value.id), + member, + ) + + _raise_static_error(node, f'Cannot take index of {node_typ} {node.value.id}') + + def visit_Module_FunctionDef_Constant(self, module: Module, function: Function, exp_type: TypeBase, node: ast.Constant) -> Expression: + del module + del function + + _not_implemented(node.kind is None, 'Constant.kind') + + if isinstance(exp_type, TypeUInt8): + if not isinstance(node.value, int): + _raise_static_error(node, 'Expected integer value') + + # FIXME: Range check + + return ConstantUInt8(exp_type, node.value) + + if isinstance(exp_type, TypeInt32): + if not isinstance(node.value, int): + _raise_static_error(node, 'Expected integer value') + + # FIXME: Range check + + return ConstantInt32(exp_type, node.value) + + if isinstance(exp_type, TypeInt64): + if not isinstance(node.value, int): + _raise_static_error(node, 'Expected integer value') + + # FIXME: Range check + + return ConstantInt64(exp_type, node.value) + + if isinstance(exp_type, TypeFloat32): + if not isinstance(node.value, (float, int, )): + _raise_static_error(node, 'Expected float value') + + # FIXME: Range check + + return ConstantFloat32(exp_type, node.value) + + if isinstance(exp_type, TypeFloat64): + if not isinstance(node.value, (float, int, )): + _raise_static_error(node, 'Expected float value') + + # FIXME: Range check + + return ConstantFloat64(exp_type, node.value) + + raise NotImplementedError(f'{node} as const for type {exp_type}') + + def visit_type(self, module: Module, node: ast.expr) -> TypeBase: + if isinstance(node, ast.Constant): + if node.value is None: + return module.types['None'] + + _raise_static_error(node, f'Unrecognized type {node.value}') + + if isinstance(node, ast.Name): + if not isinstance(node.ctx, ast.Load): + _raise_static_error(node, 'Must be load context') + + if node.id in module.types: + return module.types[node.id] + + if node.id in module.structs: + return module.structs[node.id] + + _raise_static_error(node, f'Unrecognized type {node.id}') + + if isinstance(node, ast.Tuple): + if not isinstance(node.ctx, ast.Load): + _raise_static_error(node, 'Must be load context') + + result = TypeTuple() + + offset = 0 + + for idx, elt in enumerate(node.elts): + member = TypeTupleMember(idx, self.visit_type(module, elt), offset) + + result.members.append(member) + offset += member.type.alloc_size() + + key = result.render_internal_name() + + if key not in module.types: + module.types[key] = result + constructor = TupleConstructor(result) + module.functions[constructor.name] = constructor + + return module.types[key] + + raise NotImplementedError(f'{node} as type') + +def _not_implemented(check: Any, msg: str) -> None: + if not check: + raise NotImplementedError(msg) + +def _raise_static_error(node: Union[ast.mod, ast.stmt, ast.expr], msg: str) -> NoReturn: + raise StaticError( + f'Static error on line {node.lineno}: {msg}' + ) diff --git a/phasm/typing.py b/phasm/typing.py index 4252fdb..bca76b2 100644 --- a/phasm/typing.py +++ b/phasm/typing.py @@ -9,14 +9,6 @@ class TypeBase: """ __slots__ = () - def render(self) -> str: - """ - Renders the type back to source code format - - This'll look like Python code. - """ - raise NotImplementedError(self, 'render') - def alloc_size(self) -> int: """ When allocating this type in memory, how many bytes do we need to reserve? @@ -29,27 +21,18 @@ class TypeNone(TypeBase): """ __slots__ = () - def render(self) -> str: - return 'None' - class TypeBool(TypeBase): """ The boolean type """ __slots__ = () - def render(self) -> str: - return 'bool' - class TypeUInt8(TypeBase): """ The Integer type, unsigned and 8 bits wide """ __slots__ = () - def render(self) -> str: - return 'u8' - def alloc_size(self) -> int: return 4 # Int32 under the hood @@ -59,9 +42,6 @@ class TypeInt32(TypeBase): """ __slots__ = () - def render(self) -> str: - return 'i32' - def alloc_size(self) -> int: return 4 @@ -71,9 +51,6 @@ class TypeInt64(TypeBase): """ __slots__ = () - def render(self) -> str: - return 'i64' - def alloc_size(self) -> int: return 8 @@ -83,9 +60,6 @@ class TypeFloat32(TypeBase): """ __slots__ = () - def render(self) -> str: - return 'f32' - def alloc_size(self) -> int: return 4 @@ -95,9 +69,6 @@ class TypeFloat64(TypeBase): """ __slots__ = () - def render(self) -> str: - return 'f64' - def alloc_size(self) -> int: return 8 @@ -107,9 +78,6 @@ class TypeBytes(TypeBase): """ __slots__ = () - def render(self) -> str: - return 'bytes' - class TypeTupleMember: """ Represents a tuple member @@ -130,12 +98,11 @@ class TypeTuple(TypeBase): def __init__(self) -> None: self.members = [] - def render(self) -> str: - mems = ', '.join(x.type.render() for x in self.members) - return f'({mems}, )' - def render_internal_name(self) -> str: - mems = '@'.join(x.type.render() for x in self.members) + """ + Generates an internal name for this tuple + """ + mems = '@'.join('?' for x in self.members) assert ' ' not in mems, 'Not implement yet: subtuples' return f'tuple@{mems}' @@ -179,26 +146,6 @@ class TypeStruct(TypeBase): return None - def render(self) -> str: - """ - Renders the type back to source code format - - This'll look like Python code. - """ - return self.name - - def render_definition(self) -> str: - """ - Renders the definition back to source code format - - This'll look like Python code. - """ - result = f'class {self.name}:\n' - for mem in self.members: - result += f' {mem.name}: {mem.type.render()}\n' - - return result - def alloc_size(self) -> int: return sum( x.type.alloc_size() diff --git a/phasm/utils.py b/phasm/utils.py deleted file mode 100644 index cc9e0a4..0000000 --- a/phasm/utils.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -Utility functions -""" - -import ast - -from .ourlang import OurVisitor, Module - -def our_process(source: str, input_name: str) -> Module: - """ - Processes the python code into web assembly code - """ - res = ast.parse(source, input_name) - - our_visitor = OurVisitor() - return our_visitor.visit_Module(res) diff --git a/phasm/wasm.py b/phasm/wasm.py index 3e5cff4..5fa2506 100644 --- a/phasm/wasm.py +++ b/phasm/wasm.py @@ -168,9 +168,6 @@ class ModuleMemory(WatSerializable): self.data = data def to_wat(self) -> str: - """ - Renders this memory as WebAssembly Text - """ data = ''.join( f'\\{x:02x}' for x in self.data diff --git a/pylintrc b/pylintrc index eb6abcb..bfa8c51 100644 --- a/pylintrc +++ b/pylintrc @@ -1,2 +1,2 @@ [MASTER] -disable=C0122,R0903,R0913 +disable=C0122,R0903,R0911,R0912,R0913,R0915,R1710,W0223 diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index d19c697..cfad8ed 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -14,8 +14,9 @@ import wasmer_compiler_cranelift import wasmtime -from phasm.utils import our_process -from phasm.compiler import module +from phasm.codestyle import phasm_render +from phasm.compiler import phasm_compile +from phasm.parser import phasm_parse DASHES = '-' * 16 @@ -62,9 +63,8 @@ class Suite: """ WebAssembly test suite """ - def __init__(self, code_py, test_name): + def __init__(self, code_py): self.code_py = code_py - self.test_name = test_name def run_code(self, *args, runtime='pywasm3', imports=None): """ @@ -73,15 +73,15 @@ class Suite: Returned is an object with the results set """ - our_module = our_process(self.code_py, self.test_name) + phasm_module = phasm_parse(self.code_py) # Check if code formatting works - assert self.code_py == '\n' + our_module.render() # \n for formatting in tests + assert self.code_py == '\n' + phasm_render(phasm_module) # \n for formatting in tests # Compile - wasm_module = module(our_module) + wasm_module = phasm_compile(phasm_module) - # Render as text + # Render as WebAssembly text code_wat = wasm_module.to_wat() sys.stderr.write(f'{DASHES} Assembly {DASHES}\n') diff --git a/tests/integration/test_fib.py b/tests/integration/test_fib.py index b27297d..20e7e63 100644 --- a/tests/integration/test_fib.py +++ b/tests/integration/test_fib.py @@ -25,6 +25,6 @@ def testEntry() -> i32: return fib(40) """ - result = Suite(code_py, 'test_fib').run_code() + result = Suite(code_py).run_code() assert 102334155 == result.returned_value diff --git a/tests/integration/test_runtime_checks.py b/tests/integration/test_runtime_checks.py index abe7081..6a70032 100644 --- a/tests/integration/test_runtime_checks.py +++ b/tests/integration/test_runtime_checks.py @@ -10,6 +10,6 @@ def testEntry(f: bytes) -> u8: return f[50] """ - result = Suite(code_py, 'test_call').run_code(b'Short', b'Long' * 100) + result = Suite(code_py).run_code(b'Short', b'Long' * 100) assert 0 == result.returned_value diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index 9bb3139..51586af 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -19,7 +19,7 @@ def testEntry() -> {type_}: return 13 """ - result = Suite(code_py, 'test_return').run_code() + result = Suite(code_py).run_code() assert 13 == result.returned_value assert TYPE_MAP[type_] == type(result.returned_value) @@ -33,7 +33,7 @@ def testEntry() -> {type_}: return 10 + 3 """ - result = Suite(code_py, 'test_addition').run_code() + result = Suite(code_py).run_code() assert 13 == result.returned_value assert TYPE_MAP[type_] == type(result.returned_value) @@ -47,7 +47,7 @@ def testEntry() -> {type_}: return 10 - 3 """ - result = Suite(code_py, 'test_addition').run_code() + result = Suite(code_py).run_code() assert 7 == result.returned_value assert TYPE_MAP[type_] == type(result.returned_value) @@ -61,7 +61,7 @@ def testEntry() -> {type_}: return sqrt(25) """ - result = Suite(code_py, 'test_addition').run_code() + result = Suite(code_py).run_code() assert 5 == result.returned_value assert TYPE_MAP[type_] == type(result.returned_value) @@ -75,7 +75,7 @@ def testEntry(a: {type_}) -> {type_}: return a """ - result = Suite(code_py, 'test_return').run_code(125) + result = Suite(code_py).run_code(125) assert 125 == result.returned_value assert TYPE_MAP[type_] == type(result.returned_value) @@ -89,7 +89,7 @@ def testEntry(a: i32) -> i64: return a """ - result = Suite(code_py, 'test_return').run_code(125) + result = Suite(code_py).run_code(125) assert 125 == result.returned_value assert [] == result.log_int32_list @@ -103,7 +103,7 @@ def testEntry(a: i32, b: i64) -> i64: return a + b """ - result = Suite(code_py, 'test_return').run_code(125, 100) + result = Suite(code_py).run_code(125, 100) assert 225 == result.returned_value assert [] == result.log_int32_list @@ -117,7 +117,7 @@ def testEntry(a: f32) -> f64: return a """ - result = Suite(code_py, 'test_return').run_code(125.5) + result = Suite(code_py).run_code(125.5) assert 125.5 == result.returned_value assert [] == result.log_int32_list @@ -131,7 +131,7 @@ def testEntry(a: f32, b: f64) -> f64: return a + b """ - result = Suite(code_py, 'test_return').run_code(125.5, 100.25) + result = Suite(code_py).run_code(125.5, 100.25) assert 225.75 == result.returned_value assert [] == result.log_int32_list @@ -145,7 +145,7 @@ def testEntry() -> i32: return +523 """ - result = Suite(code_py, 'test_addition').run_code() + result = Suite(code_py).run_code() assert 523 == result.returned_value assert [] == result.log_int32_list @@ -159,7 +159,7 @@ def testEntry() -> i32: return -19 """ - result = Suite(code_py, 'test_addition').run_code() + result = Suite(code_py).run_code() assert -19 == result.returned_value assert [] == result.log_int32_list @@ -177,7 +177,7 @@ def testEntry(a: i32) -> i32: """ exp_result = 15 if inp > 10 else 3 - suite = Suite(code_py, 'test_return') + suite = Suite(code_py) result = suite.run_code(inp) assert exp_result == result.returned_value @@ -198,7 +198,7 @@ def testEntry(a: i32) -> i32: return -1 # Required due to function type """ - suite = Suite(code_py, 'test_return') + suite = Suite(code_py) assert 10 == suite.run_code(20).returned_value assert 10 == suite.run_code(10).returned_value @@ -208,6 +208,30 @@ def testEntry(a: i32) -> i32: assert 0 == suite.run_code(0).returned_value assert 0 == suite.run_code(-1).returned_value +@pytest.mark.integration_test +def test_if_nested(): + code_py = """ +@exported +def testEntry(a: i32, b: i32) -> i32: + if a > 11: + if b > 11: + return 3 + + return 2 + + if b > 11: + return 1 + + return 0 +""" + + suite = Suite(code_py) + + assert 3 == suite.run_code(20, 20).returned_value + assert 2 == suite.run_code(20, 10).returned_value + assert 1 == suite.run_code(10, 20).returned_value + assert 0 == suite.run_code(10, 10).returned_value + @pytest.mark.integration_test def test_call_pre_defined(): code_py = """ @@ -219,7 +243,7 @@ def testEntry() -> i32: return helper(10, 3) """ - result = Suite(code_py, 'test_call').run_code() + result = Suite(code_py).run_code() assert 13 == result.returned_value assert [] == result.log_int32_list @@ -235,7 +259,7 @@ def helper(left: i32, right: i32) -> i32: return left - right """ - result = Suite(code_py, 'test_call').run_code() + result = Suite(code_py).run_code() assert 7 == result.returned_value assert [] == result.log_int32_list @@ -252,7 +276,7 @@ def helper(left: {type_}, right: {type_}) -> {type_}: return left - right """ - result = Suite(code_py, 'test_call').run_code() + result = Suite(code_py).run_code() assert 22 == result.returned_value assert TYPE_MAP[type_] == type(result.returned_value) @@ -268,7 +292,7 @@ def testEntry() -> i32: return a """ - result = Suite(code_py, 'test_call').run_code() + result = Suite(code_py).run_code() assert 8947 == result.returned_value assert [] == result.log_int32_list @@ -288,7 +312,7 @@ def helper(cv: CheckedValue) -> {type_}: return cv.value """ - result = Suite(code_py, 'test_call').run_code() + result = Suite(code_py).run_code() assert 23 == result.returned_value assert [] == result.log_int32_list @@ -309,7 +333,7 @@ def helper(shape: Rectangle) -> i32: return shape.height + shape.width + shape.border """ - result = Suite(code_py, 'test_call').run_code() + result = Suite(code_py).run_code() assert 252 == result.returned_value assert [] == result.log_int32_list @@ -330,7 +354,7 @@ def helper(shape1: Rectangle, shape2: Rectangle) -> i32: return shape1.height + shape1.width + shape1.border + shape2.height + shape2.width + shape2.border """ - result = Suite(code_py, 'test_call').run_code() + result = Suite(code_py).run_code() assert 545 == result.returned_value assert [] == result.log_int32_list @@ -347,7 +371,7 @@ def helper(vector: ({type_}, {type_}, {type_}, )) -> {type_}: return vector[0] + vector[1] + vector[2] """ - result = Suite(code_py, 'test_call').run_code() + result = Suite(code_py).run_code() assert 161 == result.returned_value assert TYPE_MAP[type_] == type(result.returned_value) @@ -363,7 +387,7 @@ def helper(v: (f32, f32, f32, )) -> f32: return sqrt(v[0] * v[0] + v[1] * v[1] + v[2] * v[2]) """ - result = Suite(code_py, 'test_call').run_code() + result = Suite(code_py).run_code() assert 3.74 < result.returned_value < 3.75 assert [] == result.log_int32_list @@ -376,7 +400,7 @@ def testEntry(f: bytes) -> bytes: return f """ - result = Suite(code_py, 'test_call').run_code(b'This is a test') + result = Suite(code_py).run_code(b'This is a test') assert 4 == result.returned_value @@ -388,7 +412,7 @@ def testEntry(f: bytes) -> i32: return len(f) """ - result = Suite(code_py, 'test_call').run_code(b'This is another test') + result = Suite(code_py).run_code(b'This is another test') assert 20 == result.returned_value @@ -400,7 +424,7 @@ def testEntry(f: bytes) -> u8: return f[8] """ - result = Suite(code_py, 'test_call').run_code(b'This is another test') + result = Suite(code_py).run_code(b'This is another test') assert 0x61 == result.returned_value @@ -413,7 +437,7 @@ def testEntry() -> i32x4: return (51, 153, 204, 0, ) """ - result = Suite(code_py, 'test_rgb2hsl').run_code() + result = Suite(code_py).run_code() assert (1, 2, 3, 0) == result.returned_value @@ -432,7 +456,7 @@ def testEntry() -> i32: def helper(mul: int) -> int: return 4238 * mul - result = Suite(code_py, 'test_imported').run_code( + result = Suite(code_py).run_code( runtime='wasmer', imports={ 'helper': helper, diff --git a/tests/integration/test_static_checking.py b/tests/integration/test_static_checking.py index ad43a7c..69d0724 100644 --- a/tests/integration/test_static_checking.py +++ b/tests/integration/test_static_checking.py @@ -1,7 +1,7 @@ import pytest -from phasm.utils import our_process -from phasm.ourlang import StaticError +from phasm.parser import phasm_parse +from phasm.exceptions import StaticError @pytest.mark.integration_test @pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) @@ -12,7 +12,7 @@ def helper(a: {type_}) -> (i32, i32, ): """ with pytest.raises(StaticError, match=f'Static error on line 3: Expected \\(i32, i32, \\), a is actually {type_}'): - our_process(code_py, 'test') + phasm_parse(code_py) @pytest.mark.integration_test @pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) @@ -26,7 +26,7 @@ def testEntry(arg: Struct) -> (i32, i32, ): """ with pytest.raises(StaticError, match=f'Static error on line 6: Expected \\(i32, i32, \\), arg.param is actually {type_}'): - our_process(code_py, 'test') + phasm_parse(code_py) @pytest.mark.integration_test @pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) @@ -37,7 +37,7 @@ def testEntry(arg: ({type_}, )) -> (i32, i32, ): """ with pytest.raises(StaticError, match=f'Static error on line 3: Expected \\(i32, i32, \\), arg\\[0\\] is actually {type_}'): - our_process(code_py, 'test') + phasm_parse(code_py) @pytest.mark.integration_test @pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) @@ -52,4 +52,4 @@ def testEntry() -> (i32, i32, ): """ with pytest.raises(StaticError, match=f'Static error on line 7: Expected \\(i32, i32, \\), helper actually returns {type_}'): - our_process(code_py, 'test') + phasm_parse(code_py) From a83858aca74e56532708c8f4b05a53c243f9fb5e Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 9 Jul 2022 14:22:38 +0200 Subject: [PATCH 48/73] Adds u32 and u64 Also, adds some range checks to constants. --- phasm/codestyle.py | 11 ++++++- phasm/compiler.py | 50 ++++++++++++++++++++++++++++++-- phasm/ourlang.py | 28 +++++++++++++++++- phasm/parser.py | 32 +++++++++++++++++--- phasm/typing.py | 25 ++++++++++++++++ tests/integration/test_simple.py | 22 +++++++++----- 6 files changed, 152 insertions(+), 16 deletions(-) diff --git a/phasm/codestyle.py b/phasm/codestyle.py index 5d217f5..93853dd 100644 --- a/phasm/codestyle.py +++ b/phasm/codestyle.py @@ -29,6 +29,12 @@ def type_(inp: typing.TypeBase) -> str: if isinstance(inp, typing.TypeUInt8): return 'u8' + if isinstance(inp, typing.TypeUInt32): + return 'u32' + + if isinstance(inp, typing.TypeUInt64): + return 'u64' + if isinstance(inp, typing.TypeInt32): return 'i32' @@ -71,7 +77,10 @@ def expression(inp: ourlang.Expression) -> str: """ Render: A Phasm expression """ - if isinstance(inp, (ourlang.ConstantUInt8, ourlang.ConstantInt32, ourlang.ConstantInt64, )): + if isinstance(inp, ( + ourlang.ConstantUInt8, ourlang.ConstantUInt32, ourlang.ConstantUInt64, + ourlang.ConstantInt32, ourlang.ConstantInt64, + )): return str(inp.value) if isinstance(inp, (ourlang.ConstantFloat32, ourlang.ConstantFloat64, )): diff --git a/phasm/compiler.py b/phasm/compiler.py index 4ad4bd2..778d11a 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -11,6 +11,8 @@ Statements = Generator[wasm.Statement, None, None] LOAD_STORE_TYPE_MAP = { typing.TypeUInt8: 'i32', + typing.TypeUInt32: 'i32', + typing.TypeUInt64: 'i64', typing.TypeInt32: 'i32', typing.TypeInt64: 'i64', typing.TypeFloat32: 'f32', @@ -39,6 +41,12 @@ def type_(inp: typing.TypeBase) -> wasm.WasmType: # So we need to store more memory per byte return wasm.WasmTypeInt32() + if isinstance(inp, typing.TypeUInt32): + return wasm.WasmTypeInt32() + + if isinstance(inp, typing.TypeUInt64): + return wasm.WasmTypeInt64() + if isinstance(inp, typing.TypeInt32): return wasm.WasmTypeInt32() @@ -66,14 +74,28 @@ OPERATOR_MAP = { '==': 'eq', } -I32_OPERATOR_MAP = { # TODO: Introduce UInt32 type +U32_OPERATOR_MAP = { + '<': 'lt_u', + '>': 'gt_u', + '<=': 'le_u', + '>=': 'ge_u', +} + +U64_OPERATOR_MAP = { + '<': 'lt_u', + '>': 'gt_u', + '<=': 'le_u', + '>=': 'ge_u', +} + +I32_OPERATOR_MAP = { '<': 'lt_s', '>': 'gt_s', '<=': 'le_s', '>=': 'ge_s', } -I64_OPERATOR_MAP = { # TODO: Introduce UInt32 type +I64_OPERATOR_MAP = { '<': 'lt_s', '>': 'gt_s', '<=': 'le_s', @@ -88,6 +110,14 @@ def expression(inp: ourlang.Expression) -> Statements: yield wasm.Statement('i32.const', str(inp.value)) return + if isinstance(inp, ourlang.ConstantUInt32): + yield wasm.Statement('i32.const', str(inp.value)) + return + + if isinstance(inp, ourlang.ConstantUInt64): + yield wasm.Statement('i64.const', str(inp.value)) + return + if isinstance(inp, ourlang.ConstantInt32): yield wasm.Statement('i32.const', str(inp.value)) return @@ -112,6 +142,20 @@ def expression(inp: ourlang.Expression) -> Statements: yield from expression(inp.left) yield from expression(inp.right) + if isinstance(inp.type, typing.TypeUInt32): + if operator := OPERATOR_MAP.get(inp.operator, None): + yield wasm.Statement(f'i32.{operator}') + return + if operator := U32_OPERATOR_MAP.get(inp.operator, None): + yield wasm.Statement(f'i32.{operator}') + return + if isinstance(inp.type, typing.TypeUInt64): + if operator := OPERATOR_MAP.get(inp.operator, None): + yield wasm.Statement(f'i64.{operator}') + return + if operator := U64_OPERATOR_MAP.get(inp.operator, None): + yield wasm.Statement(f'i64.{operator}') + return if isinstance(inp.type, typing.TypeInt32): if operator := OPERATOR_MAP.get(inp.operator, None): yield wasm.Statement(f'i32.{operator}') @@ -288,7 +332,7 @@ def function(inp: ourlang.Function) -> wasm.Function: for y in inp.statements for x in statement(y) ] - locals_ = [] # TODO + locals_ = [] # FIXME: Implement function locals, if required return wasm.Function( inp.name, diff --git a/phasm/ourlang.py b/phasm/ourlang.py index 1903328..ef9e37a 100644 --- a/phasm/ourlang.py +++ b/phasm/ourlang.py @@ -12,7 +12,7 @@ from .typing import ( TypeBase, TypeNone, TypeBool, - TypeUInt8, + TypeUInt8, TypeUInt32, TypeUInt64, TypeInt32, TypeInt64, TypeFloat32, TypeFloat64, TypeBytes, @@ -49,6 +49,30 @@ class ConstantUInt8(Constant): super().__init__(type_) self.value = value +class ConstantUInt32(Constant): + """ + An UInt32 constant value expression within a statement + """ + __slots__ = ('value', ) + + value: int + + def __init__(self, type_: TypeUInt32, value: int) -> None: + super().__init__(type_) + self.value = value + +class ConstantUInt64(Constant): + """ + An UInt64 constant value expression within a statement + """ + __slots__ = ('value', ) + + value: int + + def __init__(self, type_: TypeUInt64, value: int) -> None: + super().__init__(type_) + self.value = value + class ConstantInt32(Constant): """ An Int32 constant value expression within a statement @@ -317,6 +341,8 @@ class Module: self.types = { 'None': TypeNone(), 'u8': TypeUInt8(), + 'u32': TypeUInt32(), + 'u64': TypeUInt64(), 'i32': TypeInt32(), 'i64': TypeInt64(), 'f32': TypeFloat32(), diff --git a/phasm/parser.py b/phasm/parser.py index dda06e6..a810910 100644 --- a/phasm/parser.py +++ b/phasm/parser.py @@ -8,6 +8,8 @@ import ast from .typing import ( TypeBase, TypeUInt8, + TypeUInt32, + TypeUInt64, TypeInt32, TypeInt64, TypeFloat32, @@ -30,7 +32,8 @@ from .ourlang import ( Expression, AccessBytesIndex, AccessStructMember, AccessTupleMember, BinaryOp, - ConstantFloat32, ConstantFloat64, ConstantInt32, ConstantInt64, ConstantUInt8, + ConstantFloat32, ConstantFloat64, ConstantInt32, ConstantInt64, + ConstantUInt8, ConstantUInt32, ConstantUInt64, FunctionCall, StructConstructor, TupleConstructor, UnaryOp, VariableReference, @@ -485,15 +488,35 @@ class OurVisitor: if not isinstance(node.value, int): _raise_static_error(node, 'Expected integer value') - # FIXME: Range check + if node.value < 0 or node.value > 255: + _raise_static_error(node, 'Integer value out of range') return ConstantUInt8(exp_type, node.value) + if isinstance(exp_type, TypeUInt32): + if not isinstance(node.value, int): + _raise_static_error(node, 'Expected integer value') + + if node.value < 0 or node.value > 4294967295: + _raise_static_error(node, 'Integer value out of range') + + return ConstantUInt32(exp_type, node.value) + + if isinstance(exp_type, TypeUInt64): + if not isinstance(node.value, int): + _raise_static_error(node, 'Expected integer value') + + if node.value < 0 or node.value > 18446744073709551615: + _raise_static_error(node, 'Integer value out of range') + + return ConstantUInt64(exp_type, node.value) + if isinstance(exp_type, TypeInt32): if not isinstance(node.value, int): _raise_static_error(node, 'Expected integer value') - # FIXME: Range check + if node.value < -2147483648 or node.value > 2147483647: + _raise_static_error(node, 'Integer value out of range') return ConstantInt32(exp_type, node.value) @@ -501,7 +524,8 @@ class OurVisitor: if not isinstance(node.value, int): _raise_static_error(node, 'Expected integer value') - # FIXME: Range check + if node.value < -9223372036854775808 or node.value > 9223372036854775807: + _raise_static_error(node, 'Integer value out of range') return ConstantInt64(exp_type, node.value) diff --git a/phasm/typing.py b/phasm/typing.py index bca76b2..71a7549 100644 --- a/phasm/typing.py +++ b/phasm/typing.py @@ -30,12 +30,37 @@ class TypeBool(TypeBase): class TypeUInt8(TypeBase): """ The Integer type, unsigned and 8 bits wide + + Note that under the hood we need to use i32 to represent + these values in expressions. So we need to add some operations + to make sure the math checks out. + + So while this does save bytes in memory, it may not actually + speed up or improve your code. """ __slots__ = () def alloc_size(self) -> int: return 4 # Int32 under the hood +class TypeUInt32(TypeBase): + """ + The Integer type, unsigned and 32 bits wide + """ + __slots__ = () + + def alloc_size(self) -> int: + return 4 + +class TypeUInt64(TypeBase): + """ + The Integer type, unsigned and 64 bits wide + """ + __slots__ = () + + def alloc_size(self) -> int: + return 8 + class TypeInt32(TypeBase): """ The Integer type, signed and 32 bits wide diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index 51586af..4cba3e6 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -4,14 +4,22 @@ from .helpers import Suite TYPE_MAP = { 'u8': int, + 'u32': int, + 'u64': int, 'i32': int, 'i64': int, 'f32': float, 'f64': float, } +COMPLETE_SIMPLE_TYPES = [ + 'u32', 'u64', + 'i32', 'i64', + 'f32', 'f64', +] + @pytest.mark.integration_test -@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64', 'u8']) +@pytest.mark.parametrize('type_', TYPE_MAP.keys()) def test_return(type_): code_py = f""" @exported @@ -25,7 +33,7 @@ def testEntry() -> {type_}: assert TYPE_MAP[type_] == type(result.returned_value) @pytest.mark.integration_test -@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) +@pytest.mark.parametrize('type_', COMPLETE_SIMPLE_TYPES) def test_addition(type_): code_py = f""" @exported @@ -39,7 +47,7 @@ def testEntry() -> {type_}: assert TYPE_MAP[type_] == type(result.returned_value) @pytest.mark.integration_test -@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) +@pytest.mark.parametrize('type_', COMPLETE_SIMPLE_TYPES) def test_subtraction(type_): code_py = f""" @exported @@ -67,7 +75,7 @@ def testEntry() -> {type_}: assert TYPE_MAP[type_] == type(result.returned_value) @pytest.mark.integration_test -@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64', 'u8']) +@pytest.mark.parametrize('type_', TYPE_MAP.keys()) def test_arg(type_): code_py = f""" @exported @@ -265,7 +273,7 @@ def helper(left: i32, right: i32) -> i32: assert [] == result.log_int32_list @pytest.mark.integration_test -@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) +@pytest.mark.parametrize('type_', COMPLETE_SIMPLE_TYPES) def test_call_with_expression(type_): code_py = f""" @exported @@ -298,7 +306,7 @@ def testEntry() -> i32: assert [] == result.log_int32_list @pytest.mark.integration_test -@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64', 'u8']) +@pytest.mark.parametrize('type_', TYPE_MAP.keys()) def test_struct_0(type_): code_py = f""" class CheckedValue: @@ -360,7 +368,7 @@ def helper(shape1: Rectangle, shape2: Rectangle) -> i32: assert [] == result.log_int32_list @pytest.mark.integration_test -@pytest.mark.parametrize('type_', ['i32', 'i64', 'f32', 'f64']) +@pytest.mark.parametrize('type_', COMPLETE_SIMPLE_TYPES) def test_tuple_simple(type_): code_py = f""" @exported From fe864e6b9d0803419d677c3341ae7ab4d8795d35 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Wed, 20 Jul 2022 20:39:55 +0200 Subject: [PATCH 49/73] Example code --- examples/crc32.py | 88 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 examples/crc32.py diff --git a/examples/crc32.py b/examples/crc32.py new file mode 100644 index 0000000..87b6912 --- /dev/null +++ b/examples/crc32.py @@ -0,0 +1,88 @@ +# #include // uint32_t, uint8_t +# +# uint32_t CRC32(const uint8_t data[], size_t data_length) { +# uint32_t crc32 = 0xFFFFFFFFu; +# +# for (size_t i = 0; i < data_length; i++) { +# const uint32_t lookupIndex = (crc32 ^ data[i]) & 0xff; +# crc32 = (crc32 >> 8) ^ CRCTable[lookupIndex]; // CRCTable is an array of 256 32-bit constants +# } +# +# // Finalize the CRC-32 value by inverting all the bits +# crc32 ^= 0xFFFFFFFFu; +# return crc32; +# } + +_CRC32_Table: [u32] = [ + 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, + 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, + 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, + 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24, + 0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B, + 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384, + 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54, + 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B, + 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, + 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, + 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5, + 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA, + 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, + 0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A, + 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, + 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595, + 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, + 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957, + 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, + 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198, + 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927, + 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, + 0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8, + 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7, + 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096, + 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789, + 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, + 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, + 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, + 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6, + 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, + 0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829, + 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, + 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93, + 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, + 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C, + 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, + 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC, + 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, + 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, + 0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652, + 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D, + 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D, + 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982, + 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, + 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622, + 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, + 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED, + 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, + 0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F, + 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, + 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0, + 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F, + 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540, + 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, + 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F, + 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE, + 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1, + 0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321, + 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E, + 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, + 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, + 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, + 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351, +] + +def _crc32_f(val: u32, byt: u8) -> u32: + return (val >> 8) ^ _CRC32_Table[(val ^ byt) & 0xFF] + +@exported +def crc32(data: bytes) -> u32: + return 0xFFFFFFFF ^ foldli(_crc32_f, 0xFFFFFFFF, data) From b42ae275b982d3c38c4d5ea285da3d2d013c5675 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Thu, 4 Aug 2022 20:09:01 +0200 Subject: [PATCH 50/73] Start on new allocator --- phasm/compiler.py | 73 +++++++++++++++++ tests/integration/test_stdlib_alloc.py | 106 +++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 tests/integration/test_stdlib_alloc.py diff --git a/phasm/compiler.py b/phasm/compiler.py index 778d11a..0b456a2 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -360,6 +360,8 @@ def module(inp: ourlang.Module) -> wasm.Module: result.functions = [ _generate____new_reference___(inp), + _generate_stdlib_alloc___init__(inp), + _generate_stdlib_alloc___alloc__(inp), _generate____access_bytes_index___(inp), ] + [ function(x) @@ -392,6 +394,77 @@ def _generate____new_reference___(mod: ourlang.Module) -> wasm.Function: ], ) +def _generate_stdlib_alloc___init__(mod: ourlang.Module) -> wasm.Function: + return wasm.Function( + 'stdlib.alloc.__init__', + 'stdlib.alloc.__init__', + [], + [], + wasm.WasmTypeNone(), + [ + wasm.Statement('i32.const', '0x00'), + wasm.Statement('i32.load'), + wasm.Statement('i32.const', '0xA1C0'), + wasm.Statement('i32.eq'), + wasm.Statement('if', comment='Already set up'), + wasm.Statement('return'), + wasm.Statement('end'), + + wasm.Statement('i32.const', '0x04', comment='Reserved'), + wasm.Statement('i32.const', '0x000000'), + wasm.Statement('i32.store'), + + wasm.Statement('i32.const', '0x08', comment='Address of next free block'), + wasm.Statement('i32.const', '0x000000'), + wasm.Statement('i32.store'), + + wasm.Statement('i32.const', '0x0C', comment='Reserved'), + wasm.Statement('i32.const', '0x000000'), + wasm.Statement('i32.store'), + + wasm.Statement('i32.const', '0x00', comment='Done setting up'), + wasm.Statement('i32.const', '0xA1C0'), + wasm.Statement('i32.store'), + ], + ) + +def _generate_stdlib_alloc___alloc__(mod: ourlang.Module) -> wasm.Function: + return wasm.Function( + 'stdlib.alloc.__alloc__', + 'stdlib.alloc.__alloc__', + [ + ('alloc_size', type_(mod.types['i32']), ), + ], + [ + ('result', type_(mod.types['i32']), ), + ], + type_(mod.types['i32']), + [ + wasm.Statement('i32.const', '0'), + wasm.Statement('i32.load'), + wasm.Statement('i32.const', '0xA1C0'), + wasm.Statement('i32.ne'), + wasm.Statement('if', comment='Failed to set up'), + wasm.Statement('unreachable'), + wasm.Statement('end'), + + # wasm.Statement('local.get', '$alloc_size'), + # wasm.Statement('call', '$stdlib.alloc.__find_space__'), + wasm.Statement('i32.const', '0x16', comment='STUB'), + wasm.Statement('local.set', '$result'), + + # Store block size + wasm.Statement('local.get', '$result'), + wasm.Statement('local.get', '$alloc_size'), + wasm.Statement('i32.store', 'offset=0'), + + # Return address of the allocated bytes + wasm.Statement('local.get', '$result'), + wasm.Statement('i32.const', '0x04'), + wasm.Statement('i32.add'), + ], + ) + def _generate____access_bytes_index___(mod: ourlang.Module) -> wasm.Function: return wasm.Function( '___access_bytes_index___', diff --git a/tests/integration/test_stdlib_alloc.py b/tests/integration/test_stdlib_alloc.py new file mode 100644 index 0000000..52ccfa4 --- /dev/null +++ b/tests/integration/test_stdlib_alloc.py @@ -0,0 +1,106 @@ +import sys + +import pytest + +import wasm3 + +from phasm.compiler import phasm_compile +from phasm.parser import phasm_parse + +from .helpers import DASHES, wat2wasm, _dump_memory, _write_numbered_lines + +def setup_interpreter(code_phasm): + phasm_module = phasm_parse(code_phasm) + wasm_module = phasm_compile(phasm_module) + code_wat = wasm_module.to_wat() + + sys.stderr.write(f'{DASHES} Assembly {DASHES}\n') + _write_numbered_lines(code_wat) + + code_wasm = wat2wasm(code_wat) + + env = wasm3.Environment() + mod = env.parse_module(code_wasm) + + rtime = env.new_runtime(1024 * 1024) + rtime.load(mod) + + return rtime + +@pytest.mark.integration_test +def test___init__(): + code_py = """ +@exported +def testEntry() -> u8: + return 13 +""" + + rtime = setup_interpreter(code_py) + + for idx in range(128): + rtime.get_memory(0)[idx] = idx + + sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') + _dump_memory(rtime.get_memory(0)) + + rtime.find_function('stdlib.alloc.__init__')() + + sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') + _dump_memory(rtime.get_memory(0)) + + memory = rtime.get_memory(0).tobytes() + + assert ( + b'\xC0\xA1\x00\x00' + b'\x00\x00\x00\x00' + b'\x00\x00\x00\x00' + b'\x00\x00\x00\x00' + b'\x10\x11\x12\x13' # Untouched because unused + ) == memory[0:20] + +@pytest.mark.integration_test +def test___alloc___no_init(): + code_py = """ +@exported +def testEntry() -> u8: + return 13 +""" + + rtime = setup_interpreter(code_py) + + for idx in range(128): + rtime.get_memory(0)[idx] = idx + + sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') + _dump_memory(rtime.get_memory(0)) + + with pytest.raises(RuntimeError, match='unreachable executed'): + rtime.find_function('stdlib.alloc.__alloc__')(32) + +@pytest.mark.integration_test +def test___alloc___ok(): + code_py = """ +@exported +def testEntry() -> u8: + return 13 +""" + + rtime = setup_interpreter(code_py) + + for idx in range(128): + rtime.get_memory(0)[idx] = idx + + sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') + _dump_memory(rtime.get_memory(0)) + + offset = rtime.find_function('stdlib.alloc.__init__')() + offset = rtime.find_function('stdlib.alloc.__alloc__')(32) + + sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') + _dump_memory(rtime.get_memory(0)) + + memory = rtime.get_memory(0).tobytes() + + assert b'\xC0\xA1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' == memory[0:12] + + assert 0 == offset From e03f038cf937316f878d9be115c00cc0f2385edb Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Thu, 4 Aug 2022 20:51:59 +0200 Subject: [PATCH 51/73] More work on allocator --- phasm/compiler.py | 69 ++++++++++++++++++++++---- tests/integration/test_stdlib_alloc.py | 14 ++++-- 2 files changed, 70 insertions(+), 13 deletions(-) diff --git a/phasm/compiler.py b/phasm/compiler.py index 0b456a2..b74b1c7 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -361,6 +361,7 @@ def module(inp: ourlang.Module) -> wasm.Module: result.functions = [ _generate____new_reference___(inp), _generate_stdlib_alloc___init__(inp), + _generate_stdlib_alloc___find_free_block__(inp), _generate_stdlib_alloc___alloc__(inp), _generate____access_bytes_index___(inp), ] + [ @@ -394,6 +395,9 @@ def _generate____new_reference___(mod: ourlang.Module) -> wasm.Function: ], ) +STDLIB_ALLOC__FREE_BLOCK = '0x08' +STDLIB_ALLOC__UNALLOC_ADDR = '0x0C' + def _generate_stdlib_alloc___init__(mod: ourlang.Module) -> wasm.Function: return wasm.Function( 'stdlib.alloc.__init__', @@ -411,15 +415,17 @@ def _generate_stdlib_alloc___init__(mod: ourlang.Module) -> wasm.Function: wasm.Statement('end'), wasm.Statement('i32.const', '0x04', comment='Reserved'), - wasm.Statement('i32.const', '0x000000'), + wasm.Statement('i32.const', '0x00000000'), wasm.Statement('i32.store'), - wasm.Statement('i32.const', '0x08', comment='Address of next free block'), - wasm.Statement('i32.const', '0x000000'), + wasm.Statement('i32.const', STDLIB_ALLOC__FREE_BLOCK, + comment='Address of next free block'), + wasm.Statement('i32.const', '0x00000000'), wasm.Statement('i32.store'), - wasm.Statement('i32.const', '0x0C', comment='Reserved'), - wasm.Statement('i32.const', '0x000000'), + wasm.Statement('i32.const', STDLIB_ALLOC__UNALLOC_ADDR, + comment='Address of first unallocated byte'), + wasm.Statement('i32.const', '0x10'), wasm.Statement('i32.store'), wasm.Statement('i32.const', '0x00', comment='Done setting up'), @@ -428,6 +434,30 @@ def _generate_stdlib_alloc___init__(mod: ourlang.Module) -> wasm.Function: ], ) +def _generate_stdlib_alloc___find_free_block__(mod: ourlang.Module) -> wasm.Function: + return wasm.Function( + 'stdlib.alloc.__find_free_block__', + 'stdlib.alloc.__find_free_block__', + [ + ('alloc_size', type_(mod.types['i32']), ), + ], + [ + ('result', type_(mod.types['i32']), ), + ], + type_(mod.types['i32']), + [ + wasm.Statement('i32.const', STDLIB_ALLOC__FREE_BLOCK), + wasm.Statement('i32.load'), + wasm.Statement('i32.const', '0x00000000'), + wasm.Statement('i32.eq'), + wasm.Statement('if'), + wasm.Statement('i32.const', '0x00000000'), + wasm.Statement('return'), + wasm.Statement('end'), + wasm.Statement('unreachable'), + ], + ) + def _generate_stdlib_alloc___alloc__(mod: ourlang.Module) -> wasm.Function: return wasm.Function( 'stdlib.alloc.__alloc__', @@ -444,15 +474,36 @@ def _generate_stdlib_alloc___alloc__(mod: ourlang.Module) -> wasm.Function: wasm.Statement('i32.load'), wasm.Statement('i32.const', '0xA1C0'), wasm.Statement('i32.ne'), - wasm.Statement('if', comment='Failed to set up'), + wasm.Statement('if', comment='Failed to set up or memory corruption'), wasm.Statement('unreachable'), wasm.Statement('end'), - # wasm.Statement('local.get', '$alloc_size'), - # wasm.Statement('call', '$stdlib.alloc.__find_space__'), - wasm.Statement('i32.const', '0x16', comment='STUB'), + wasm.Statement('local.get', '$alloc_size'), + wasm.Statement('call', '$stdlib.alloc.__find_free_block__'), wasm.Statement('local.set', '$result'), + # Check if there was a free block + wasm.Statement('local.get', '$result'), + wasm.Statement('i32.const', '0x00000000'), + wasm.Statement('i32.eq'), + wasm.Statement('if', comment='No free block found'), + + # Use unallocated space + wasm.Statement('i32.const', STDLIB_ALLOC__UNALLOC_ADDR), + + wasm.Statement('i32.const', STDLIB_ALLOC__UNALLOC_ADDR), + wasm.Statement('i32.load'), + wasm.Statement('local.tee', '$result'), + + # Updated unalloc pointer (address already set on stack) + wasm.Statement('i32.const', '0x04'), # Struct size + wasm.Statement('i32.add'), + wasm.Statement('local.get', '$alloc_size'), + wasm.Statement('i32.add'), + wasm.Statement('i32.store', 'offset=0'), + + wasm.Statement('end'), + # Store block size wasm.Statement('local.get', '$result'), wasm.Statement('local.get', '$alloc_size'), diff --git a/tests/integration/test_stdlib_alloc.py b/tests/integration/test_stdlib_alloc.py index 52ccfa4..80c3065 100644 --- a/tests/integration/test_stdlib_alloc.py +++ b/tests/integration/test_stdlib_alloc.py @@ -54,7 +54,7 @@ def testEntry() -> u8: b'\xC0\xA1\x00\x00' b'\x00\x00\x00\x00' b'\x00\x00\x00\x00' - b'\x00\x00\x00\x00' + b'\x10\x00\x00\x00' b'\x10\x11\x12\x13' # Untouched because unused ) == memory[0:20] @@ -93,8 +93,10 @@ def testEntry() -> u8: sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') _dump_memory(rtime.get_memory(0)) - offset = rtime.find_function('stdlib.alloc.__init__')() - offset = rtime.find_function('stdlib.alloc.__alloc__')(32) + rtime.find_function('stdlib.alloc.__init__')() + offset0 = rtime.find_function('stdlib.alloc.__alloc__')(32) + offset1 = rtime.find_function('stdlib.alloc.__alloc__')(32) + offset2 = rtime.find_function('stdlib.alloc.__alloc__')(32) sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') _dump_memory(rtime.get_memory(0)) @@ -103,4 +105,8 @@ def testEntry() -> u8: assert b'\xC0\xA1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' == memory[0:12] - assert 0 == offset + assert 0x14 == offset0 + assert 0x38 == offset1 + assert 0x5C == offset2 + + assert False From fea817ca00dd3e161acd5adad89645b286896ab9 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Thu, 4 Aug 2022 21:18:53 +0200 Subject: [PATCH 52/73] Added more robust and easy way to generate WASM --- phasm/compiler.py | 139 +++++++++++++------------ phasm/wasmeasy.py | 69 ++++++++++++ tests/integration/test_stdlib_alloc.py | 2 - 3 files changed, 140 insertions(+), 70 deletions(-) create mode 100644 phasm/wasmeasy.py diff --git a/phasm/compiler.py b/phasm/compiler.py index b74b1c7..52624a8 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -6,6 +6,9 @@ from typing import Generator from . import ourlang from . import typing from . import wasm +from . import wasmeasy + +from .wasmeasy import i32, i64 Statements = Generator[wasm.Statement, None, None] @@ -107,23 +110,23 @@ def expression(inp: ourlang.Expression) -> Statements: Compile: Any expression """ if isinstance(inp, ourlang.ConstantUInt8): - yield wasm.Statement('i32.const', str(inp.value)) + yield i32.const(inp.value) return if isinstance(inp, ourlang.ConstantUInt32): - yield wasm.Statement('i32.const', str(inp.value)) + yield i32.const(inp.value) return if isinstance(inp, ourlang.ConstantUInt64): - yield wasm.Statement('i64.const', str(inp.value)) + yield i64.const(inp.value) return if isinstance(inp, ourlang.ConstantInt32): - yield wasm.Statement('i32.const', str(inp.value)) + yield i32.const(inp.value) return if isinstance(inp, ourlang.ConstantInt64): - yield wasm.Statement('i64.const', str(inp.value)) + yield i64.const(inp.value) return if isinstance(inp, ourlang.ConstantFloat32): @@ -196,7 +199,7 @@ def expression(inp: ourlang.Expression) -> Statements: if isinstance(inp.type, typing.TypeInt32): if inp.operator == 'len': if isinstance(inp.right.type, typing.TypeBytes): - yield wasm.Statement('i32.load') + yield i32.load() return raise NotImplementedError(expression, inp.type, inp.operator) @@ -384,19 +387,21 @@ def _generate____new_reference___(mod: ourlang.Module) -> wasm.Function: ], type_(mod.types['i32']), [ - wasm.Statement('i32.const', '0'), - wasm.Statement('i32.const', '0'), - wasm.Statement('i32.load'), + i32.const(0), + i32.const(0), + i32.load(), wasm.Statement('local.tee', '$result', comment='Address for this call'), wasm.Statement('local.get', '$alloc_size'), - wasm.Statement('i32.add'), - wasm.Statement('i32.store', comment='Address for the next call'), + i32.add(), + i32.store(comment='Address for the next call'), wasm.Statement('local.get', '$result'), ], ) -STDLIB_ALLOC__FREE_BLOCK = '0x08' -STDLIB_ALLOC__UNALLOC_ADDR = '0x0C' +STDLIB_ALLOC__IDENTIFIER = 0xA1C0 +STDLIB_ALLOC__RESERVED0 = 0x04 +STDLIB_ALLOC__FREE_BLOCK = 0x08 +STDLIB_ALLOC__UNALLOC_ADDR = 0x0C def _generate_stdlib_alloc___init__(mod: ourlang.Module) -> wasm.Function: return wasm.Function( @@ -406,31 +411,31 @@ def _generate_stdlib_alloc___init__(mod: ourlang.Module) -> wasm.Function: [], wasm.WasmTypeNone(), [ - wasm.Statement('i32.const', '0x00'), - wasm.Statement('i32.load'), - wasm.Statement('i32.const', '0xA1C0'), - wasm.Statement('i32.eq'), - wasm.Statement('if', comment='Already set up'), - wasm.Statement('return'), - wasm.Statement('end'), + i32.const(0), + i32.load(), + i32.const(STDLIB_ALLOC__IDENTIFIER), + i32.eq(), + *wasmeasy.if_( + wasm.Statement('return', comment='Already set up'), + ), - wasm.Statement('i32.const', '0x04', comment='Reserved'), - wasm.Statement('i32.const', '0x00000000'), - wasm.Statement('i32.store'), + i32.const(STDLIB_ALLOC__RESERVED0, comment='Reserved'), + i32.const(0), + i32.store(), - wasm.Statement('i32.const', STDLIB_ALLOC__FREE_BLOCK, + i32.const(STDLIB_ALLOC__FREE_BLOCK, comment='Address of next free block'), - wasm.Statement('i32.const', '0x00000000'), - wasm.Statement('i32.store'), + i32.const(0, comment='None to start with'), + i32.store(), - wasm.Statement('i32.const', STDLIB_ALLOC__UNALLOC_ADDR, + i32.const(STDLIB_ALLOC__UNALLOC_ADDR, comment='Address of first unallocated byte'), - wasm.Statement('i32.const', '0x10'), - wasm.Statement('i32.store'), + i32.const(0x10), + i32.store(), - wasm.Statement('i32.const', '0x00', comment='Done setting up'), - wasm.Statement('i32.const', '0xA1C0'), - wasm.Statement('i32.store'), + i32.const(0, comment='Done setting up'), + i32.const(STDLIB_ALLOC__IDENTIFIER), + i32.store(), ], ) @@ -446,14 +451,14 @@ def _generate_stdlib_alloc___find_free_block__(mod: ourlang.Module) -> wasm.Func ], type_(mod.types['i32']), [ - wasm.Statement('i32.const', STDLIB_ALLOC__FREE_BLOCK), - wasm.Statement('i32.load'), - wasm.Statement('i32.const', '0x00000000'), - wasm.Statement('i32.eq'), - wasm.Statement('if'), - wasm.Statement('i32.const', '0x00000000'), - wasm.Statement('return'), - wasm.Statement('end'), + i32.const(STDLIB_ALLOC__FREE_BLOCK), + i32.load(), + i32.const(0), + i32.eq(), + *wasmeasy.if_( + i32.const(0), + wasm.Statement('return'), + ), wasm.Statement('unreachable'), ], ) @@ -470,13 +475,13 @@ def _generate_stdlib_alloc___alloc__(mod: ourlang.Module) -> wasm.Function: ], type_(mod.types['i32']), [ - wasm.Statement('i32.const', '0'), - wasm.Statement('i32.load'), - wasm.Statement('i32.const', '0xA1C0'), - wasm.Statement('i32.ne'), - wasm.Statement('if', comment='Failed to set up or memory corruption'), - wasm.Statement('unreachable'), - wasm.Statement('end'), + i32.const(0), + i32.load(), + i32.const(STDLIB_ALLOC__IDENTIFIER), + i32.ne(), + *wasmeasy.if_( + wasm.Statement('unreachable'), + ), wasm.Statement('local.get', '$alloc_size'), wasm.Statement('call', '$stdlib.alloc.__find_free_block__'), @@ -484,35 +489,33 @@ def _generate_stdlib_alloc___alloc__(mod: ourlang.Module) -> wasm.Function: # Check if there was a free block wasm.Statement('local.get', '$result'), - wasm.Statement('i32.const', '0x00000000'), - wasm.Statement('i32.eq'), - wasm.Statement('if', comment='No free block found'), + i32.const(0), + i32.eq(), + *wasmeasy.if_( + # Use unallocated space + i32.const(STDLIB_ALLOC__UNALLOC_ADDR), - # Use unallocated space - wasm.Statement('i32.const', STDLIB_ALLOC__UNALLOC_ADDR), + i32.const(STDLIB_ALLOC__UNALLOC_ADDR), + i32.load(), + wasm.Statement('local.tee', '$result'), - wasm.Statement('i32.const', STDLIB_ALLOC__UNALLOC_ADDR), - wasm.Statement('i32.load'), - wasm.Statement('local.tee', '$result'), - - # Updated unalloc pointer (address already set on stack) - wasm.Statement('i32.const', '0x04'), # Struct size - wasm.Statement('i32.add'), - wasm.Statement('local.get', '$alloc_size'), - wasm.Statement('i32.add'), - wasm.Statement('i32.store', 'offset=0'), - - wasm.Statement('end'), + # Updated unalloc pointer (address already set on stack) + i32.const(4), # Header size + i32.add(), + wasm.Statement('local.get', '$alloc_size'), + i32.add(), + i32.store('offset=0'), + ), # Store block size wasm.Statement('local.get', '$result'), wasm.Statement('local.get', '$alloc_size'), - wasm.Statement('i32.store', 'offset=0'), + i32.store('offset=0'), # Return address of the allocated bytes wasm.Statement('local.get', '$result'), - wasm.Statement('i32.const', '0x04'), - wasm.Statement('i32.add'), + i32.const(4), # Header size + i32.add(), ], ) @@ -530,7 +533,7 @@ def _generate____access_bytes_index___(mod: ourlang.Module) -> wasm.Function: [ wasm.Statement('local.get', '$ofs'), wasm.Statement('local.get', '$byt'), - wasm.Statement('i32.load'), + i32.load(), wasm.Statement('i32.lt_u'), wasm.Statement('if', comment='$ofs < len($byt)'), diff --git a/phasm/wasmeasy.py b/phasm/wasmeasy.py new file mode 100644 index 0000000..01c5d0c --- /dev/null +++ b/phasm/wasmeasy.py @@ -0,0 +1,69 @@ +""" +Helper functions to quickly generate WASM code +""" +from typing import List, Optional + +import functools + +from . import wasm + +#pylint: disable=C0103,C0115,C0116,R0201,R0902 + +class Prefix_inn_fnn: + def __init__(self, prefix: str) -> None: + self.prefix = prefix + + # 6.5.5. Memory Instructions + self.load = functools.partial(wasm.Statement, f'{self.prefix}.load') + self.store = functools.partial(wasm.Statement, f'{self.prefix}.store') + + # 6.5.6. Numeric Instructions + self.clz = functools.partial(wasm.Statement, f'{self.prefix}.clz') + self.ctz = functools.partial(wasm.Statement, f'{self.prefix}.ctz') + self.popcnt = functools.partial(wasm.Statement, f'{self.prefix}.popcnt') + self.add = functools.partial(wasm.Statement, f'{self.prefix}.add') + self.sub = functools.partial(wasm.Statement, f'{self.prefix}.sub') + self.mul = functools.partial(wasm.Statement, f'{self.prefix}.mul') + self.div_s = functools.partial(wasm.Statement, f'{self.prefix}.div_s') + self.div_u = functools.partial(wasm.Statement, f'{self.prefix}.div_u') + self.rem_s = functools.partial(wasm.Statement, f'{self.prefix}.rem_s') + self.rem_u = functools.partial(wasm.Statement, f'{self.prefix}.rem_u') + self.and_ = functools.partial(wasm.Statement, f'{self.prefix}.and') + self.or_ = functools.partial(wasm.Statement, f'{self.prefix}.or') + self.xor = functools.partial(wasm.Statement, f'{self.prefix}.xor') + self.shl = functools.partial(wasm.Statement, f'{self.prefix}.shl') + self.shr_s = functools.partial(wasm.Statement, f'{self.prefix}.shr_s') + self.shr_u = functools.partial(wasm.Statement, f'{self.prefix}.shr_u') + self.rotl = functools.partial(wasm.Statement, f'{self.prefix}.rotl') + self.rotr = functools.partial(wasm.Statement, f'{self.prefix}.rotr') + + self.eqz = functools.partial(wasm.Statement, f'{self.prefix}.eqz') + self.eq = functools.partial(wasm.Statement, f'{self.prefix}.eq') + self.ne = functools.partial(wasm.Statement, f'{self.prefix}.ne') + self.lt_s = functools.partial(wasm.Statement, f'{self.prefix}.lt_s') + self.lt_u = functools.partial(wasm.Statement, f'{self.prefix}.lt_u') + self.gt_s = functools.partial(wasm.Statement, f'{self.prefix}.gt_s') + self.gt_u = functools.partial(wasm.Statement, f'{self.prefix}.gt_u') + self.le_s = functools.partial(wasm.Statement, f'{self.prefix}.le_s') + self.le_u = functools.partial(wasm.Statement, f'{self.prefix}.le_u') + self.ge_s = functools.partial(wasm.Statement, f'{self.prefix}.ge_s') + self.ge_u = functools.partial(wasm.Statement, f'{self.prefix}.ge_u') + + def const(self, value: int, comment: Optional[str] = None) -> wasm.Statement: + return wasm.Statement(f'{self.prefix}.const', f'0x{value:08x}', comment=comment) + +i32 = Prefix_inn_fnn('i32') +i64 = Prefix_inn_fnn('i64') + +class Block: + def __init__(self, start: str) -> None: + self.start = start + + def __call__(self, *statements: wasm.Statement) -> List[wasm.Statement]: + return [ + wasm.Statement('if'), + *statements, + wasm.Statement('end'), + ] + +if_ = Block('if') diff --git a/tests/integration/test_stdlib_alloc.py b/tests/integration/test_stdlib_alloc.py index 80c3065..8763e46 100644 --- a/tests/integration/test_stdlib_alloc.py +++ b/tests/integration/test_stdlib_alloc.py @@ -108,5 +108,3 @@ def testEntry() -> u8: assert 0x14 == offset0 assert 0x38 == offset1 assert 0x5C == offset2 - - assert False From a5c68065d7d3619d261cd0aefefa6cf0cb2c5890 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 6 Aug 2022 13:44:11 +0200 Subject: [PATCH 53/73] More ideas about easy code generation --- phasm/stdlib/__init__.py | 0 phasm/stdlib/alloc.py | 129 +++++++++++++++++++++++++++++++++++++++ phasm/wasmeasy.py | 75 ++++++++++++++++++++++- pylintrc | 3 + 4 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 phasm/stdlib/__init__.py create mode 100644 phasm/stdlib/alloc.py diff --git a/phasm/stdlib/__init__.py b/phasm/stdlib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/phasm/stdlib/alloc.py b/phasm/stdlib/alloc.py new file mode 100644 index 0000000..02c2d48 --- /dev/null +++ b/phasm/stdlib/alloc.py @@ -0,0 +1,129 @@ +""" +stdlib: Memory allocation +""" +from phasm.wasmeasy import Generator, func_wrapper + +IDENTIFIER = 0xA1C0 + +ADR_IDENTIFIER = 0 +ADR_RESERVED0 = ADR_IDENTIFIER + 4 +ADR_FREE_BLOCK_PTR = ADR_RESERVED0 + 4 +ADR_UNALLOC_PTR = ADR_FREE_BLOCK_PTR + 4 + +@func_wrapper +def __init__(g: Generator) -> None: + """ + Initializes the memory so we can allocate it + """ + + # Check if the memory is already initialized + g.i32.const(0) + g.i32.load() + g.i32.const(IDENTIFIER) + g.i32.eq() + + with g.if_(): + # Already initialized, return without any changes + g.return_() + + # Store the first reserved i32 + g.i32.const(ADR_RESERVED0) + g.i32.const(0) + g.i32.store() + + # Store the pointer towards the first free block + # In this case, 0 since we haven't freed any blocks yet + g.i32.const(ADR_FREE_BLOCK_PTR) + g.i32.const(0) + g.i32.store() + + # Store the pointer towards the first unallocated block + # In this case the end of the stdlib.alloc header at the start + g.i32.const(ADR_UNALLOC_PTR) + g.i32.const(0x10) + g.i32.store() + + # Store that we've initialized the memory + g.i32.const(0) + g.i32.const(IDENTIFIER) + g.i32.store() + + +@func_wrapper +def __find_free_block__(g: Generator) -> None: + # Find out if we've freed any blocks at all so far + g.i32.const(ADR_FREE_BLOCK_PTR) + g.i32.load() + g.i32.const(0) + g.i32.eq() + + with g.if_(): + g.i32.const(0) + g.return_() + + g.unreachable() + +@func_wrapper +def __alloc__(g: Generator) -> None: + g.i32.const(ADR_IDENTIFIER) + g.i32.load() + g.i32.const(IDENTIFIER) + g.i32.ne() + with g.if_(): + g.unreachable() + +# def _generate_stdlib_alloc___alloc__(mod: ourlang.Module) -> wasm.Function: +# return wasm.Function( +# 'stdlib.alloc.__alloc__', +# 'stdlib.alloc.__alloc__', +# [ +# ('alloc_size', type_(mod.types['i32']), ), +# ], +# [ +# ('result', type_(mod.types['i32']), ), +# ], +# type_(mod.types['i32']), +# [ +# i32.const(0), +# i32.load(), +# i32.const(STDLIB_ALLOC__IDENTIFIER), +# i32.ne(), +# *wasmeasy.if_( +# wasm.Statement('unreachable'), +# ), +# +# wasm.Statement('local.get', '$alloc_size'), +# wasm.Statement('call', '$stdlib.alloc.__find_free_block__'), +# wasm.Statement('local.set', '$result'), +# +# # Check if there was a free block +# wasm.Statement('local.get', '$result'), +# i32.const(0), +# i32.eq(), +# *wasmeasy.if_( +# # Use unallocated space +# i32.const(STDLIB_ALLOC__ADR_UNALLOC_PTR), +# +# i32.const(STDLIB_ALLOC__ADR_UNALLOC_PTR), +# i32.load(), +# wasm.Statement('local.tee', '$result'), +# +# # Updated unalloc pointer (address already set on stack) +# i32.const(4), # Header size +# i32.add(), +# wasm.Statement('local.get', '$alloc_size'), +# i32.add(), +# i32.store('offset=0'), +# ), +# +# # Store block size +# wasm.Statement('local.get', '$result'), +# wasm.Statement('local.get', '$alloc_size'), +# i32.store('offset=0'), +# +# # Return address of the allocated bytes +# wasm.Statement('local.get', '$result'), +# i32.const(4), # Header size +# i32.add(), +# ], +# ) diff --git a/phasm/wasmeasy.py b/phasm/wasmeasy.py index 01c5d0c..3cf8ae1 100644 --- a/phasm/wasmeasy.py +++ b/phasm/wasmeasy.py @@ -1,7 +1,7 @@ """ Helper functions to quickly generate WASM code """ -from typing import List, Optional +from typing import Any, List, Optional import functools @@ -67,3 +67,76 @@ class Block: ] if_ = Block('if') + +class Generator_i32: + def __init__(self, generator: 'Generator') -> None: + self.generator = generator + + # 2.4.1. Numeric Instructions + self.eq = functools.partial(self.generator.add, 'i32.eq') + self.ne = functools.partial(self.generator.add, 'i32.ne') + + # 2.4.4. Memory Instructions + self.load = functools.partial(self.generator.add, 'i32.load') + self.store = functools.partial(self.generator.add, 'i32.store') + + def const(self, value: int, comment: Optional[str] = None) -> None: + self.generator.add('i32.const', f'0x{value:08x}', comment=comment) + +class GeneratorBlock: + def __init__(self, generator: 'Generator', name: str) -> None: + self.generator = generator + self.name = name + + def __enter__(self) -> None: + self.generator.add(self.name) + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + if not exc_type: + self.generator.add('end') + +class Generator: + def __init__(self) -> None: + self.statements: List[wasm.Statement] = [] + + self.i32 = Generator_i32(self) + + # 2.4.5 Control Instructions + self.nop = functools.partial(self.add, 'nop') + self.unreachable = functools.partial(self.add, 'unreachable') + # block + # loop + self.if_ = functools.partial(GeneratorBlock, self, 'if') + # br + # br_if + # br_table + self.return_ = functools.partial(self.add, 'return') + # call + # call_indirect + + def add(self, name: str, *args: str, comment: Optional[str] = None) -> None: + self.statements.append(wasm.Statement(name, *args, comment=comment)) + +def func_wrapper(func: Any) -> wasm.Function: + assert func.__module__.startswith('phasm.') + + assert Generator is func.__annotations__['g'] + + params: Any = [] + + locals_: Any = [] + + assert None is func.__annotations__['return'] + return_type = wasm.WasmTypeNone() + + generator = Generator() + func(g=generator) + + return wasm.Function( + func.__module__[6:] + '.' + func.__name__, + func.__module__[6:] + '.' + func.__name__, + params, + locals_, + return_type, + generator.statements, + ) diff --git a/pylintrc b/pylintrc index bfa8c51..b38e5c0 100644 --- a/pylintrc +++ b/pylintrc @@ -1,2 +1,5 @@ [MASTER] disable=C0122,R0903,R0911,R0912,R0913,R0915,R1710,W0223 + +[stdlib] +good-names=g From c5e2744b3e6229111e498a7e9b0ce546683dce67 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 6 Aug 2022 14:52:57 +0200 Subject: [PATCH 54/73] Moved stdlib.alloc to the new generator --- phasm/compiler.py | 130 ++-------------------------------- phasm/stdlib/alloc.py | 122 +++++++++++++++----------------- phasm/wasmeasy.py | 75 +------------------- phasm/wasmgenerator.py | 154 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 217 insertions(+), 264 deletions(-) create mode 100644 phasm/wasmgenerator.py diff --git a/phasm/compiler.py b/phasm/compiler.py index 52624a8..5385c3a 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -8,6 +8,7 @@ from . import typing from . import wasm from . import wasmeasy +from .stdlib import alloc as stdlib_alloc from .wasmeasy import i32, i64 Statements = Generator[wasm.Statement, None, None] @@ -362,10 +363,10 @@ def module(inp: ourlang.Module) -> wasm.Module: ] result.functions = [ - _generate____new_reference___(inp), - _generate_stdlib_alloc___init__(inp), - _generate_stdlib_alloc___find_free_block__(inp), - _generate_stdlib_alloc___alloc__(inp), + _generate____new_reference___(inp), # Old allocator + stdlib_alloc.__init__, + stdlib_alloc.__find_free_block__, + stdlib_alloc.__alloc__, _generate____access_bytes_index___(inp), ] + [ function(x) @@ -398,127 +399,6 @@ def _generate____new_reference___(mod: ourlang.Module) -> wasm.Function: ], ) -STDLIB_ALLOC__IDENTIFIER = 0xA1C0 -STDLIB_ALLOC__RESERVED0 = 0x04 -STDLIB_ALLOC__FREE_BLOCK = 0x08 -STDLIB_ALLOC__UNALLOC_ADDR = 0x0C - -def _generate_stdlib_alloc___init__(mod: ourlang.Module) -> wasm.Function: - return wasm.Function( - 'stdlib.alloc.__init__', - 'stdlib.alloc.__init__', - [], - [], - wasm.WasmTypeNone(), - [ - i32.const(0), - i32.load(), - i32.const(STDLIB_ALLOC__IDENTIFIER), - i32.eq(), - *wasmeasy.if_( - wasm.Statement('return', comment='Already set up'), - ), - - i32.const(STDLIB_ALLOC__RESERVED0, comment='Reserved'), - i32.const(0), - i32.store(), - - i32.const(STDLIB_ALLOC__FREE_BLOCK, - comment='Address of next free block'), - i32.const(0, comment='None to start with'), - i32.store(), - - i32.const(STDLIB_ALLOC__UNALLOC_ADDR, - comment='Address of first unallocated byte'), - i32.const(0x10), - i32.store(), - - i32.const(0, comment='Done setting up'), - i32.const(STDLIB_ALLOC__IDENTIFIER), - i32.store(), - ], - ) - -def _generate_stdlib_alloc___find_free_block__(mod: ourlang.Module) -> wasm.Function: - return wasm.Function( - 'stdlib.alloc.__find_free_block__', - 'stdlib.alloc.__find_free_block__', - [ - ('alloc_size', type_(mod.types['i32']), ), - ], - [ - ('result', type_(mod.types['i32']), ), - ], - type_(mod.types['i32']), - [ - i32.const(STDLIB_ALLOC__FREE_BLOCK), - i32.load(), - i32.const(0), - i32.eq(), - *wasmeasy.if_( - i32.const(0), - wasm.Statement('return'), - ), - wasm.Statement('unreachable'), - ], - ) - -def _generate_stdlib_alloc___alloc__(mod: ourlang.Module) -> wasm.Function: - return wasm.Function( - 'stdlib.alloc.__alloc__', - 'stdlib.alloc.__alloc__', - [ - ('alloc_size', type_(mod.types['i32']), ), - ], - [ - ('result', type_(mod.types['i32']), ), - ], - type_(mod.types['i32']), - [ - i32.const(0), - i32.load(), - i32.const(STDLIB_ALLOC__IDENTIFIER), - i32.ne(), - *wasmeasy.if_( - wasm.Statement('unreachable'), - ), - - wasm.Statement('local.get', '$alloc_size'), - wasm.Statement('call', '$stdlib.alloc.__find_free_block__'), - wasm.Statement('local.set', '$result'), - - # Check if there was a free block - wasm.Statement('local.get', '$result'), - i32.const(0), - i32.eq(), - *wasmeasy.if_( - # Use unallocated space - i32.const(STDLIB_ALLOC__UNALLOC_ADDR), - - i32.const(STDLIB_ALLOC__UNALLOC_ADDR), - i32.load(), - wasm.Statement('local.tee', '$result'), - - # Updated unalloc pointer (address already set on stack) - i32.const(4), # Header size - i32.add(), - wasm.Statement('local.get', '$alloc_size'), - i32.add(), - i32.store('offset=0'), - ), - - # Store block size - wasm.Statement('local.get', '$result'), - wasm.Statement('local.get', '$alloc_size'), - i32.store('offset=0'), - - # Return address of the allocated bytes - wasm.Statement('local.get', '$result'), - i32.const(4), # Header size - i32.add(), - ], - ) - def _generate____access_bytes_index___(mod: ourlang.Module) -> wasm.Function: return wasm.Function( '___access_bytes_index___', diff --git a/phasm/stdlib/alloc.py b/phasm/stdlib/alloc.py index 02c2d48..73608ff 100644 --- a/phasm/stdlib/alloc.py +++ b/phasm/stdlib/alloc.py @@ -1,7 +1,7 @@ """ stdlib: Memory allocation """ -from phasm.wasmeasy import Generator, func_wrapper +from phasm.wasmgenerator import Generator, VarType_i32 as i32, func_wrapper IDENTIFIER = 0xA1C0 @@ -10,18 +10,19 @@ ADR_RESERVED0 = ADR_IDENTIFIER + 4 ADR_FREE_BLOCK_PTR = ADR_RESERVED0 + 4 ADR_UNALLOC_PTR = ADR_FREE_BLOCK_PTR + 4 -@func_wrapper +UNALLOC_PTR = ADR_UNALLOC_PTR + 4 + +@func_wrapper() def __init__(g: Generator) -> None: """ Initializes the memory so we can allocate it """ # Check if the memory is already initialized - g.i32.const(0) + g.i32.const(ADR_IDENTIFIER) g.i32.load() g.i32.const(IDENTIFIER) g.i32.eq() - with g.if_(): # Already initialized, return without any changes g.return_() @@ -40,7 +41,7 @@ def __init__(g: Generator) -> None: # Store the pointer towards the first unallocated block # In this case the end of the stdlib.alloc header at the start g.i32.const(ADR_UNALLOC_PTR) - g.i32.const(0x10) + g.i32.const(UNALLOC_PTR) g.i32.store() # Store that we've initialized the memory @@ -48,9 +49,8 @@ def __init__(g: Generator) -> None: g.i32.const(IDENTIFIER) g.i32.store() - -@func_wrapper -def __find_free_block__(g: Generator) -> None: +@func_wrapper(exported=False) +def __find_free_block__(g: Generator, alloc_size: i32) -> i32: # Find out if we've freed any blocks at all so far g.i32.const(ADR_FREE_BLOCK_PTR) g.i32.load() @@ -61,69 +61,61 @@ def __find_free_block__(g: Generator) -> None: g.i32.const(0) g.return_() + del alloc_size # TODO g.unreachable() -@func_wrapper -def __alloc__(g: Generator) -> None: + return i32('return') # To satisfy mypy + +@func_wrapper() +def __alloc__(g: Generator, alloc_size: i32) -> i32: + result = i32('result') + + # Check if the memory is already initialized g.i32.const(ADR_IDENTIFIER) g.i32.load() g.i32.const(IDENTIFIER) g.i32.ne() with g.if_(): + # Not yet initialized, or memory corruption g.unreachable() -# def _generate_stdlib_alloc___alloc__(mod: ourlang.Module) -> wasm.Function: -# return wasm.Function( -# 'stdlib.alloc.__alloc__', -# 'stdlib.alloc.__alloc__', -# [ -# ('alloc_size', type_(mod.types['i32']), ), -# ], -# [ -# ('result', type_(mod.types['i32']), ), -# ], -# type_(mod.types['i32']), -# [ -# i32.const(0), -# i32.load(), -# i32.const(STDLIB_ALLOC__IDENTIFIER), -# i32.ne(), -# *wasmeasy.if_( -# wasm.Statement('unreachable'), -# ), -# -# wasm.Statement('local.get', '$alloc_size'), -# wasm.Statement('call', '$stdlib.alloc.__find_free_block__'), -# wasm.Statement('local.set', '$result'), -# -# # Check if there was a free block -# wasm.Statement('local.get', '$result'), -# i32.const(0), -# i32.eq(), -# *wasmeasy.if_( -# # Use unallocated space -# i32.const(STDLIB_ALLOC__ADR_UNALLOC_PTR), -# -# i32.const(STDLIB_ALLOC__ADR_UNALLOC_PTR), -# i32.load(), -# wasm.Statement('local.tee', '$result'), -# -# # Updated unalloc pointer (address already set on stack) -# i32.const(4), # Header size -# i32.add(), -# wasm.Statement('local.get', '$alloc_size'), -# i32.add(), -# i32.store('offset=0'), -# ), -# -# # Store block size -# wasm.Statement('local.get', '$result'), -# wasm.Statement('local.get', '$alloc_size'), -# i32.store('offset=0'), -# -# # Return address of the allocated bytes -# wasm.Statement('local.get', '$result'), -# i32.const(4), # Header size -# i32.add(), -# ], -# ) + # Try to claim a free block + g.local.get(alloc_size) + g.call(__find_free_block__) + g.local.set(result) + + # Check if there was a free block + g.local.get(result) + g.i32.const(0) + g.i32.eq() + with g.if_(): + # No free blocks, increase allocated memory usage + + # Put the address on the stack in advance so we can store to it later + g.i32.const(ADR_UNALLOC_PTR) + + # Get the current unalloc pointer value + g.i32.const(ADR_UNALLOC_PTR) + g.i32.load() + g.local.tee(result) + + # Calculate new unalloc pointer value + g.i32.const(4) # Header size + g.i32.add() + g.local.get(alloc_size) + g.i32.add() + + # Store new unalloc pointer value (address was set on stack in advance) + g.i32.store() + + # Store block size in the header + g.local.get(result) + g.local.get(alloc_size) + g.i32.store() + + # Return address of the allocated bytes + g.local.get(result) + g.i32.const(4) # Header size + g.i32.add() + + return i32('return') # To satisfy mypy diff --git a/phasm/wasmeasy.py b/phasm/wasmeasy.py index 3cf8ae1..d0cf358 100644 --- a/phasm/wasmeasy.py +++ b/phasm/wasmeasy.py @@ -1,7 +1,7 @@ """ Helper functions to quickly generate WASM code """ -from typing import Any, List, Optional +from typing import Any, Dict, List, Optional, Type import functools @@ -67,76 +67,3 @@ class Block: ] if_ = Block('if') - -class Generator_i32: - def __init__(self, generator: 'Generator') -> None: - self.generator = generator - - # 2.4.1. Numeric Instructions - self.eq = functools.partial(self.generator.add, 'i32.eq') - self.ne = functools.partial(self.generator.add, 'i32.ne') - - # 2.4.4. Memory Instructions - self.load = functools.partial(self.generator.add, 'i32.load') - self.store = functools.partial(self.generator.add, 'i32.store') - - def const(self, value: int, comment: Optional[str] = None) -> None: - self.generator.add('i32.const', f'0x{value:08x}', comment=comment) - -class GeneratorBlock: - def __init__(self, generator: 'Generator', name: str) -> None: - self.generator = generator - self.name = name - - def __enter__(self) -> None: - self.generator.add(self.name) - - def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: - if not exc_type: - self.generator.add('end') - -class Generator: - def __init__(self) -> None: - self.statements: List[wasm.Statement] = [] - - self.i32 = Generator_i32(self) - - # 2.4.5 Control Instructions - self.nop = functools.partial(self.add, 'nop') - self.unreachable = functools.partial(self.add, 'unreachable') - # block - # loop - self.if_ = functools.partial(GeneratorBlock, self, 'if') - # br - # br_if - # br_table - self.return_ = functools.partial(self.add, 'return') - # call - # call_indirect - - def add(self, name: str, *args: str, comment: Optional[str] = None) -> None: - self.statements.append(wasm.Statement(name, *args, comment=comment)) - -def func_wrapper(func: Any) -> wasm.Function: - assert func.__module__.startswith('phasm.') - - assert Generator is func.__annotations__['g'] - - params: Any = [] - - locals_: Any = [] - - assert None is func.__annotations__['return'] - return_type = wasm.WasmTypeNone() - - generator = Generator() - func(g=generator) - - return wasm.Function( - func.__module__[6:] + '.' + func.__name__, - func.__module__[6:] + '.' + func.__name__, - params, - locals_, - return_type, - generator.statements, - ) diff --git a/phasm/wasmgenerator.py b/phasm/wasmgenerator.py new file mode 100644 index 0000000..1c8015a --- /dev/null +++ b/phasm/wasmgenerator.py @@ -0,0 +1,154 @@ +""" +Helper functions to generate WASM code by writing Python functions +""" +from typing import Any, Callable, Dict, List, Optional, Type + +import functools + +from . import wasm + +# pylint: disable=C0103,C0115,C0116,R0902 + +class VarType_Base: + wasm_type: Type[wasm.WasmType] + + def __init__(self, name: str) -> None: + self.name = name + self.name_ref = f'${name}' + +class VarType_i32(VarType_Base): + wasm_type = wasm.WasmTypeInt32 + +class Generator_i32: + def __init__(self, generator: 'Generator') -> None: + self.generator = generator + + # 2.4.1. Numeric Instructions + # ibinop + self.add = functools.partial(self.generator.add, 'i32.add') + + # irelop + self.eq = functools.partial(self.generator.add, 'i32.eq') + self.ne = functools.partial(self.generator.add, 'i32.ne') + + # 2.4.4. Memory Instructions + self.load = functools.partial(self.generator.add, 'i32.load') + self.store = functools.partial(self.generator.add, 'i32.store') + + def const(self, value: int, comment: Optional[str] = None) -> None: + self.generator.add('i32.const', f'0x{value:08x}', comment=comment) + +class Generator_Local: + def __init__(self, generator: 'Generator') -> None: + self.generator = generator + + # 2.4.3. Variable Instructions + def get(self, variable: VarType_Base, comment: Optional[str] = None) -> None: + self.generator.add('local.get', variable.name_ref, comment=comment) + + def set(self, variable: VarType_Base, comment: Optional[str] = None) -> None: + self.generator.locals.setdefault(variable.name, variable) + + self.generator.add('local.set', variable.name_ref, comment=comment) + + def tee(self, variable: VarType_Base, comment: Optional[str] = None) -> None: + self.generator.locals.setdefault(variable.name, variable) + + self.generator.add('local.tee', variable.name_ref, comment=comment) + +class GeneratorBlock: + def __init__(self, generator: 'Generator', name: str) -> None: + self.generator = generator + self.name = name + + def __enter__(self) -> None: + self.generator.add(self.name) + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + if not exc_type: + self.generator.add('end') + +class Generator: + def __init__(self) -> None: + self.statements: List[wasm.Statement] = [] + self.locals: Dict[str, VarType_Base] = {} + + self.i32 = Generator_i32(self) + + # 2.4.3 Variable Instructions + self.local = Generator_Local(self) + + # 2.4.5 Control Instructions + self.nop = functools.partial(self.add, 'nop') + self.unreachable = functools.partial(self.add, 'unreachable') + # block + # loop + self.if_ = functools.partial(GeneratorBlock, self, 'if') + # br + # br_if + # br_table + self.return_ = functools.partial(self.add, 'return') + # call + # call_indirect + + def call(self, function: wasm.Function) -> None: + self.add('call', f'${function.name}') + + def add(self, name: str, *args: str, comment: Optional[str] = None) -> None: + self.statements.append(wasm.Statement(name, *args, comment=comment)) + +def func_wrapper(exported: bool = True) -> Callable[[Any], wasm.Function]: + """ + This wrapper will execute the function and return + a wasm Function with the generated Statements + """ + def inner(func: Any) -> wasm.Function: + func_name_parts = func.__module__.split('.') + [func.__name__] + if 'phasm' == func_name_parts[0]: + func_name_parts.pop(0) + func_name = '.'.join(func_name_parts) + + annot = dict(func.__annotations__) + + # Check if we can pass the generator + assert Generator is annot.pop('g') + + # Convert return type to WasmType + annot_return = annot.pop('return') + if annot_return is None: + return_type = wasm.WasmTypeNone() + else: + assert issubclass(annot_return, VarType_Base) + return_type = annot_return.wasm_type() + + # Load the argument types, and generate instances + args: Dict[str, VarType_Base] = {} + params: List[wasm.Param] = [] + for param_name, param_type in annot.items(): + assert issubclass(param_type, VarType_Base) + + params.append((param_name, param_type.wasm_type(), )) + + args[param_name] = VarType_Base(param_name) + + # Make a generator, and run the function on that generator, + # so the Statements get added + generator = Generator() + func(g=generator, **args) + + # Check what locals were used, and define them + locals_: List[wasm.Param] = [] + for local_name, local_type in generator.locals.items(): + locals_.append((local_name, local_type.wasm_type(), )) + + # Complete function definition + return wasm.Function( + func_name, + func_name if exported else None, + params, + locals_, + return_type, + generator.statements, + ) + + return inner From 253974df24f9d9a4f5e52b526d4bdd65ec3435b4 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 6 Aug 2022 20:11:39 +0200 Subject: [PATCH 55/73] Adds runner classes to tests, implements xor for u8, u32, u64 --- Makefile | 2 +- mypy.ini | 2 + phasm/compiler.py | 13 ++ phasm/parser.py | 2 + stubs/wasm3.pyi | 23 ++++ tests/integration/runners.py | 168 +++++++++++++++++++++++++ tests/integration/test_simple.py | 14 +++ tests/integration/test_stdlib_alloc.py | 80 ++++++------ 8 files changed, 258 insertions(+), 46 deletions(-) create mode 100644 mypy.ini create mode 100644 stubs/wasm3.pyi create mode 100644 tests/integration/runners.py diff --git a/Makefile b/Makefile index 80cc868..09bbb51 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ lint: venv/.done venv/bin/pylint phasm typecheck: venv/.done - venv/bin/mypy --strict phasm + venv/bin/mypy --strict phasm tests/integration/runners.py venv/.done: requirements.txt python3.8 -m venv venv diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..3543521 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,2 @@ +[mypy] +mypy_path=stubs diff --git a/phasm/compiler.py b/phasm/compiler.py index 5385c3a..6c9ce98 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -78,11 +78,19 @@ OPERATOR_MAP = { '==': 'eq', } +U8_OPERATOR_MAP = { + # Under the hood, this is an i32 + # Implementing XOR is fine since the 3 remaining + # bytes stay zero after this operation + '^': 'xor', +} + U32_OPERATOR_MAP = { '<': 'lt_u', '>': 'gt_u', '<=': 'le_u', '>=': 'ge_u', + '^': 'xor', } U64_OPERATOR_MAP = { @@ -90,6 +98,7 @@ U64_OPERATOR_MAP = { '>': 'gt_u', '<=': 'le_u', '>=': 'ge_u', + '^': 'xor', } I32_OPERATOR_MAP = { @@ -146,6 +155,10 @@ def expression(inp: ourlang.Expression) -> Statements: yield from expression(inp.left) yield from expression(inp.right) + if isinstance(inp.type, typing.TypeUInt8): + if operator := U8_OPERATOR_MAP.get(inp.operator, None): + yield wasm.Statement(f'i32.{operator}') + return if isinstance(inp.type, typing.TypeUInt32): if operator := OPERATOR_MAP.get(inp.operator, None): yield wasm.Statement(f'i32.{operator}') diff --git a/phasm/parser.py b/phasm/parser.py index a810910..42d3bcc 100644 --- a/phasm/parser.py +++ b/phasm/parser.py @@ -242,6 +242,8 @@ class OurVisitor: operator = '-' elif isinstance(node.op, ast.Mult): operator = '*' + elif isinstance(node.op, ast.BitXor): + operator = '^' else: raise NotImplementedError(f'Operator {node.op}') diff --git a/stubs/wasm3.pyi b/stubs/wasm3.pyi new file mode 100644 index 0000000..216412c --- /dev/null +++ b/stubs/wasm3.pyi @@ -0,0 +1,23 @@ +from typing import Any, Callable + +class Module: + ... + +class Runtime: + ... + + def load(self, wasm_bin: Module) -> None: + ... + + def get_memory(self, memid: int) -> memoryview: + ... + + def find_function(self, name: str) -> Callable[[Any], Any]: + ... + +class Environment: + def new_runtime(self, mem_size: int) -> Runtime: + ... + + def parse_module(self, wasm_bin: bytes) -> Module: + ... diff --git a/tests/integration/runners.py b/tests/integration/runners.py new file mode 100644 index 0000000..0108fe4 --- /dev/null +++ b/tests/integration/runners.py @@ -0,0 +1,168 @@ +""" +Runners to help run WebAssembly code on various interpreters +""" +from typing import Any, TextIO + +import os +import subprocess +import sys +import tempfile + +from phasm.compiler import phasm_compile +from phasm.parser import phasm_parse +from phasm import ourlang +from phasm import wasm + +import wasm3 + +def wat2wasm(code_wat: str) -> bytes: + """ + Converts the given WebAssembly Assembly code into WebAssembly Binary + """ + path = os.environ.get('WAT2WASM', 'wat2wasm') + + with tempfile.NamedTemporaryFile('w+t') as input_fp: + input_fp.write(code_wat) + input_fp.flush() + + with tempfile.NamedTemporaryFile('w+b') as output_fp: + subprocess.run( + [ + path, + input_fp.name, + '-o', + output_fp.name, + ], + check=True, + ) + + output_fp.seek(0) + + return output_fp.read() + +class RunnerBase: + """ + Base class + """ + phasm_code: str + phasm_ast: ourlang.Module + wasm_ast: wasm.Module + wasm_asm: str + wasm_bin: bytes + + def __init__(self, phasm_code: str) -> None: + self.phasm_code = phasm_code + + def dump_phasm_code(self, textio: TextIO) -> None: + """ + Dumps the input Phasm code for debugging + """ + _dump_code(textio, self.phasm_code) + + def parse(self) -> None: + """ + Parses the Phasm code into an AST + """ + self.phasm_ast = phasm_parse(self.phasm_code) + + def compile_ast(self) -> None: + """ + Compiles the Phasm AST into an WebAssembly AST + """ + self.wasm_ast = phasm_compile(self.phasm_ast) + + def compile_wat(self) -> None: + """ + Compiles the WebAssembly AST into WebAssembly Assembly code + """ + self.wasm_asm = self.wasm_ast.to_wat() + + def dump_wasm_wat(self, textio: TextIO) -> None: + """ + Dumps the intermediate WebAssembly Assembly code for debugging + """ + _dump_code(textio, self.wasm_asm) + + def compile_wasm(self) -> None: + """ + Compiles the WebAssembly AST into WebAssembly Binary + """ + self.wasm_bin = wat2wasm(self.wasm_asm) + + def interpreter_setup(self) -> None: + """ + Sets up the interpreter + """ + raise NotImplementedError + + def interpreter_load(self) -> None: + """ + Loads the code into the interpreter + """ + raise NotImplementedError + + def interpreter_dump_memory(self, textio: TextIO) -> None: + """ + Dumps the interpreters memory for debugging + """ + raise NotImplementedError + + def call(self, function: str, *args: Any) -> Any: + """ + Calls the given function with the given arguments, returning the result + """ + raise NotImplementedError + +class RunnerPywasm3(RunnerBase): + """ + Implements a runner for pywasm3 + + See https://pypi.org/project/pywasm3/ + """ + env: wasm3.Environment + rtime: wasm3.Runtime + mod: wasm3.Module + + def interpreter_setup(self) -> None: + self.env = wasm3.Environment() + self.rtime = self.env.new_runtime(1024 * 1024) + + def interpreter_load(self) -> None: + self.mod = self.env.parse_module(self.wasm_bin) + self.rtime.load(self.mod) + + def interpreter_dump_memory(self, textio: TextIO) -> None: + _dump_memory(textio, self.rtime.get_memory(0)) + + def call(self, function: str, *args: Any) -> Any: + return self.rtime.find_function(function)(*args) + +def _dump_memory(textio: TextIO, mem: bytes) -> None: + line_width = 16 + + prev_line = None + skip = False + for idx in range(0, len(mem), line_width): + line = '' + for idx2 in range(0, line_width): + line += f'{mem[idx + idx2]:02X}' + if idx2 % 2 == 1: + line += ' ' + + if prev_line == line: + if not skip: + textio.write('**\n') + skip = True + else: + textio.write(f'{idx:08x} {line}\n') + + prev_line = line + +def _dump_code(textio: TextIO, text: str) -> None: + line_list = text.split('\n') + line_no_width = len(str(len(line_list))) + for line_no, line_txt in enumerate(line_list): + textio.write('{} {}\n'.format( + str(line_no + 1).zfill(line_no_width), + line_txt, + )) diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index 4cba3e6..b2d0bb2 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -60,6 +60,20 @@ def testEntry() -> {type_}: assert 7 == result.returned_value assert TYPE_MAP[type_] == type(result.returned_value) +@pytest.mark.integration_test +@pytest.mark.parametrize('type_', ['u8', 'u32', 'u64']) +def test_xor(type_): + code_py = f""" +@exported +def testEntry() -> {type_}: + return 10 ^ 3 +""" + + result = Suite(code_py).run_code() + + assert 9 == result.returned_value + assert TYPE_MAP[type_] == type(result.returned_value) + @pytest.mark.integration_test @pytest.mark.parametrize('type_', ['f32', 'f64']) def test_buildins_sqrt(type_): diff --git a/tests/integration/test_stdlib_alloc.py b/tests/integration/test_stdlib_alloc.py index 8763e46..31c6e00 100644 --- a/tests/integration/test_stdlib_alloc.py +++ b/tests/integration/test_stdlib_alloc.py @@ -2,30 +2,25 @@ import sys import pytest -import wasm3 +from .helpers import DASHES +from .runners import RunnerPywasm3 -from phasm.compiler import phasm_compile -from phasm.parser import phasm_parse +def setup_interpreter(phash_code: str) -> RunnerPywasm3: + runner = RunnerPywasm3(phash_code) -from .helpers import DASHES, wat2wasm, _dump_memory, _write_numbered_lines - -def setup_interpreter(code_phasm): - phasm_module = phasm_parse(code_phasm) - wasm_module = phasm_compile(phasm_module) - code_wat = wasm_module.to_wat() + runner.parse() + runner.compile_ast() + runner.compile_wat() + runner.compile_wasm() + runner.interpreter_setup() + runner.interpreter_load() + sys.stderr.write(f'{DASHES} Phasm {DASHES}\n') + runner.dump_phasm_code(sys.stderr) sys.stderr.write(f'{DASHES} Assembly {DASHES}\n') - _write_numbered_lines(code_wat) + runner.dump_wasm_wat(sys.stderr) - code_wasm = wat2wasm(code_wat) - - env = wasm3.Environment() - mod = env.parse_module(code_wasm) - - rtime = env.new_runtime(1024 * 1024) - rtime.load(mod) - - return rtime + return runner @pytest.mark.integration_test def test___init__(): @@ -35,20 +30,22 @@ def testEntry() -> u8: return 13 """ - rtime = setup_interpreter(code_py) + runner = setup_interpreter(code_py) + memory = runner.rtime.get_memory(0) + # Garbage in the memory so we can test for it for idx in range(128): - rtime.get_memory(0)[idx] = idx + memory[idx] = idx sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') - _dump_memory(rtime.get_memory(0)) + runner.interpreter_dump_memory(sys.stderr) - rtime.find_function('stdlib.alloc.__init__')() + runner.call('stdlib.alloc.__init__') - sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') - _dump_memory(rtime.get_memory(0)) + sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n') + runner.interpreter_dump_memory(sys.stderr) - memory = rtime.get_memory(0).tobytes() + memory = memory.tobytes() assert ( b'\xC0\xA1\x00\x00' @@ -66,16 +63,13 @@ def testEntry() -> u8: return 13 """ - rtime = setup_interpreter(code_py) - - for idx in range(128): - rtime.get_memory(0)[idx] = idx + runner = setup_interpreter(code_py) sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') - _dump_memory(rtime.get_memory(0)) + runner.interpreter_dump_memory(sys.stderr) with pytest.raises(RuntimeError, match='unreachable executed'): - rtime.find_function('stdlib.alloc.__alloc__')(32) + runner.call('stdlib.alloc.__alloc__', 32) @pytest.mark.integration_test def test___alloc___ok(): @@ -85,23 +79,19 @@ def testEntry() -> u8: return 13 """ - rtime = setup_interpreter(code_py) - - for idx in range(128): - rtime.get_memory(0)[idx] = idx + runner = setup_interpreter(code_py) + memory = runner.rtime.get_memory(0) sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') - _dump_memory(rtime.get_memory(0)) + runner.interpreter_dump_memory(sys.stderr) - rtime.find_function('stdlib.alloc.__init__')() - offset0 = rtime.find_function('stdlib.alloc.__alloc__')(32) - offset1 = rtime.find_function('stdlib.alloc.__alloc__')(32) - offset2 = rtime.find_function('stdlib.alloc.__alloc__')(32) + runner.call('stdlib.alloc.__init__') + offset0 = runner.call('stdlib.alloc.__alloc__', 32) + offset1 = runner.call('stdlib.alloc.__alloc__', 32) + offset2 = runner.call('stdlib.alloc.__alloc__', 32) - sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') - _dump_memory(rtime.get_memory(0)) - - memory = rtime.get_memory(0).tobytes() + sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n') + runner.interpreter_dump_memory(sys.stderr) assert b'\xC0\xA1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' == memory[0:12] From 58424cf2a052d47ea9bf3a4733dca83da76b6f76 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 6 Aug 2022 20:50:43 +0200 Subject: [PATCH 56/73] Adds runner for pywasm --- stubs/pywasm/__init__.pyi | 14 ++++++ stubs/pywasm/binary.pyi | 6 +++ stubs/pywasm/execution.pyi | 10 +++++ stubs/pywasm/option.pyi | 2 + tests/integration/runners.py | 62 ++++++++++++++++++++++++-- tests/integration/test_stdlib_alloc.py | 20 +++++---- 6 files changed, 101 insertions(+), 13 deletions(-) create mode 100644 stubs/pywasm/__init__.pyi create mode 100644 stubs/pywasm/binary.pyi create mode 100644 stubs/pywasm/execution.pyi create mode 100644 stubs/pywasm/option.pyi diff --git a/stubs/pywasm/__init__.pyi b/stubs/pywasm/__init__.pyi new file mode 100644 index 0000000..4d94109 --- /dev/null +++ b/stubs/pywasm/__init__.pyi @@ -0,0 +1,14 @@ +from typing import Any, Dict, List, Optional, Union + +from . import binary +from . import option +from . import execution + +class Runtime: + store: execution.Store + + def __init__(self, module: binary.Module, imps: Optional[Dict[str, Any]] = None, opts: Optional[option.Option] = None): + ... + + def exec(self, name: str, args: List[Union[int, float]]) -> Any: + ... diff --git a/stubs/pywasm/binary.pyi b/stubs/pywasm/binary.pyi new file mode 100644 index 0000000..3af65a0 --- /dev/null +++ b/stubs/pywasm/binary.pyi @@ -0,0 +1,6 @@ +from typing import BinaryIO + +class Module: + @classmethod + def from_reader(cls, reader: BinaryIO) -> 'Module': + ... diff --git a/stubs/pywasm/execution.pyi b/stubs/pywasm/execution.pyi new file mode 100644 index 0000000..54db4fb --- /dev/null +++ b/stubs/pywasm/execution.pyi @@ -0,0 +1,10 @@ +from typing import List + +class Result: + ... + +class MemoryInstance: + data: bytearray + +class Store: + memory_list: List[MemoryInstance] diff --git a/stubs/pywasm/option.pyi b/stubs/pywasm/option.pyi new file mode 100644 index 0000000..fd461ee --- /dev/null +++ b/stubs/pywasm/option.pyi @@ -0,0 +1,2 @@ +class Option: + ... diff --git a/tests/integration/runners.py b/tests/integration/runners.py index 0108fe4..5d26797 100644 --- a/tests/integration/runners.py +++ b/tests/integration/runners.py @@ -1,20 +1,21 @@ """ Runners to help run WebAssembly code on various interpreters """ -from typing import Any, TextIO +from typing import Any, Iterable, TextIO +import io import os import subprocess -import sys import tempfile +import pywasm.binary +import wasm3 + from phasm.compiler import phasm_compile from phasm.parser import phasm_parse from phasm import ourlang from phasm import wasm -import wasm3 - def wat2wasm(code_wat: str) -> bytes: """ Converts the given WebAssembly Assembly code into WebAssembly Binary @@ -101,6 +102,18 @@ class RunnerBase: """ raise NotImplementedError + def interpreter_write_memory(self, offset: int, data: Iterable[int]) -> None: + """ + Writes into the interpreters memory + """ + raise NotImplementedError + + def interpreter_read_memory(self, offset: int, length: int) -> bytes: + """ + Reads from the interpreters memory + """ + raise NotImplementedError + def interpreter_dump_memory(self, textio: TextIO) -> None: """ Dumps the interpreters memory for debugging @@ -113,6 +126,37 @@ class RunnerBase: """ raise NotImplementedError +class RunnerPywasm(RunnerBase): + """ + Implements a runner for pywasm + + See https://pypi.org/project/pywasm/ + """ + module: pywasm.binary.Module + runtime: pywasm.Runtime + + def interpreter_setup(self) -> None: + # Nothing to set up + pass + + def interpreter_load(self) -> None: + bytesio = io.BytesIO(self.wasm_bin) + self.module = pywasm.binary.Module.from_reader(bytesio) + self.runtime = pywasm.Runtime(self.module, {}, None) + + def interpreter_write_memory(self, offset: int, data: Iterable[int]) -> None: + for idx, byt in enumerate(data): + self.runtime.store.memory_list[0].data[offset + idx] = byt + + def interpreter_read_memory(self, offset: int, length: int) -> bytes: + return self.runtime.store.memory_list[0].data[offset:length] + + def interpreter_dump_memory(self, textio: TextIO) -> None: + _dump_memory(textio, self.runtime.store.memory_list[0].data) + + def call(self, function: str, *args: Any) -> Any: + return self.runtime.exec(function, [*args]) + class RunnerPywasm3(RunnerBase): """ Implements a runner for pywasm3 @@ -131,6 +175,16 @@ class RunnerPywasm3(RunnerBase): self.mod = self.env.parse_module(self.wasm_bin) self.rtime.load(self.mod) + def interpreter_write_memory(self, offset: int, data: Iterable[int]) -> None: + memory = self.rtime.get_memory(0) + + for idx, byt in enumerate(data): + memory[offset + idx] = byt # type: ignore + + def interpreter_read_memory(self, offset: int, length: int) -> bytes: + memory = self.rtime.get_memory(0) + return memory[offset:length].tobytes() + def interpreter_dump_memory(self, textio: TextIO) -> None: _dump_memory(textio, self.rtime.get_memory(0)) diff --git a/tests/integration/test_stdlib_alloc.py b/tests/integration/test_stdlib_alloc.py index 31c6e00..850c87f 100644 --- a/tests/integration/test_stdlib_alloc.py +++ b/tests/integration/test_stdlib_alloc.py @@ -31,11 +31,9 @@ def testEntry() -> u8: """ runner = setup_interpreter(code_py) - memory = runner.rtime.get_memory(0) # Garbage in the memory so we can test for it - for idx in range(128): - memory[idx] = idx + runner.interpreter_write_memory(0, range(128)) sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') runner.interpreter_dump_memory(sys.stderr) @@ -45,15 +43,13 @@ def testEntry() -> u8: sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n') runner.interpreter_dump_memory(sys.stderr) - memory = memory.tobytes() - assert ( b'\xC0\xA1\x00\x00' b'\x00\x00\x00\x00' b'\x00\x00\x00\x00' b'\x10\x00\x00\x00' b'\x10\x11\x12\x13' # Untouched because unused - ) == memory[0:20] + ) == runner.interpreter_read_memory(0, 20) @pytest.mark.integration_test def test___alloc___no_init(): @@ -68,7 +64,7 @@ def testEntry() -> u8: sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') runner.interpreter_dump_memory(sys.stderr) - with pytest.raises(RuntimeError, match='unreachable executed'): + with pytest.raises(Exception, match='unreachable'): runner.call('stdlib.alloc.__alloc__', 32) @pytest.mark.integration_test @@ -80,7 +76,6 @@ def testEntry() -> u8: """ runner = setup_interpreter(code_py) - memory = runner.rtime.get_memory(0) sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') runner.interpreter_dump_memory(sys.stderr) @@ -93,7 +88,14 @@ def testEntry() -> u8: sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n') runner.interpreter_dump_memory(sys.stderr) - assert b'\xC0\xA1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' == memory[0:12] + assert ( + b'\xC0\xA1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x7C\x00\x00\x00' + b'\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x00\x00\x00\x00' + ) == runner.interpreter_read_memory(0, 0x60) assert 0x14 == offset0 assert 0x38 == offset1 From 0edd04c207116a8ba215f4fa3920bfa552de8e3d Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sun, 7 Aug 2022 14:12:37 +0200 Subject: [PATCH 57/73] Adds RunnerWasmtime --- tests/integration/runners.py | 62 ++++++++++++++++++++++++++ tests/integration/test_stdlib_alloc.py | 6 +-- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/tests/integration/runners.py b/tests/integration/runners.py index 5d26797..c1350ff 100644 --- a/tests/integration/runners.py +++ b/tests/integration/runners.py @@ -3,6 +3,7 @@ Runners to help run WebAssembly code on various interpreters """ from typing import Any, Iterable, TextIO +import ctypes import io import os import subprocess @@ -10,6 +11,7 @@ import tempfile import pywasm.binary import wasm3 +import wasmtime from phasm.compiler import phasm_compile from phasm.parser import phasm_parse @@ -191,6 +193,66 @@ class RunnerPywasm3(RunnerBase): def call(self, function: str, *args: Any) -> Any: return self.rtime.find_function(function)(*args) +class RunnerWasmtime(RunnerBase): + """ + Implements a runner for wasmtime + + See https://pypi.org/project/wasmtime/ + """ + store: wasmtime.Store + module: wasmtime.Module + instance: wasmtime.Instance + + def interpreter_setup(self) -> None: + self.store = wasmtime.Store() + + def interpreter_load(self) -> None: + self.module = wasmtime.Module(self.store.engine, self.wasm_bin) + self.instance = wasmtime.Instance(self.store, self.module, []) + + def interpreter_write_memory(self, offset: int, data: Iterable[int]) -> None: + exports = self.instance.exports(self.store) + memory = exports['memory'] + assert isinstance(memory, wasmtime.Memory) # type hint + + data_ptr = memory.data_ptr(self.store) + data_len = memory.data_len(self.store) + + idx = offset + for byt in data: + assert idx < data_len + data_ptr[idx] = ctypes.c_ubyte(byt) + idx += 1 + + def interpreter_read_memory(self, offset: int, length: int) -> bytes: + exports = self.instance.exports(self.store) + memory = exports['memory'] + assert isinstance(memory, wasmtime.Memory) # type hint + + data_ptr = memory.data_ptr(self.store) + data_len = memory.data_len(self.store) + + raw = ctypes.string_at(data_ptr, data_len) + + return raw[offset:length] + + def interpreter_dump_memory(self, textio: TextIO) -> None: + exports = self.instance.exports(self.store) + memory = exports['memory'] + assert isinstance(memory, wasmtime.Memory) # type hint + + data_ptr = memory.data_ptr(self.store) + data_len = memory.data_len(self.store) + + _dump_memory(textio, ctypes.string_at(data_ptr, data_len)) + + def call(self, function: str, *args: Any) -> Any: + exports = self.instance.exports(self.store) + func = exports[function] + assert isinstance(func, wasmtime.Func) + + return func(self.store, *args) + def _dump_memory(textio: TextIO, mem: bytes) -> None: line_width = 16 diff --git a/tests/integration/test_stdlib_alloc.py b/tests/integration/test_stdlib_alloc.py index 850c87f..7e960d7 100644 --- a/tests/integration/test_stdlib_alloc.py +++ b/tests/integration/test_stdlib_alloc.py @@ -3,10 +3,10 @@ import sys import pytest from .helpers import DASHES -from .runners import RunnerPywasm3 +from .runners import RunnerPywasm3 as Runner -def setup_interpreter(phash_code: str) -> RunnerPywasm3: - runner = RunnerPywasm3(phash_code) +def setup_interpreter(phash_code: str) -> Runner: + runner = Runner(phash_code) runner.parse() runner.compile_ast() From 41b47e43d64eae64fb0b86fe6cb31e1c06f997dd Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sun, 7 Aug 2022 14:46:20 +0200 Subject: [PATCH 58/73] Implements RunnerWasmer, makes uses of its wat2wasm --- tests/integration/runners.py | 81 +++++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 29 deletions(-) diff --git a/tests/integration/runners.py b/tests/integration/runners.py index c1350ff..bb027f5 100644 --- a/tests/integration/runners.py +++ b/tests/integration/runners.py @@ -5,12 +5,10 @@ from typing import Any, Iterable, TextIO import ctypes import io -import os -import subprocess -import tempfile import pywasm.binary import wasm3 +import wasmer import wasmtime from phasm.compiler import phasm_compile @@ -18,31 +16,6 @@ from phasm.parser import phasm_parse from phasm import ourlang from phasm import wasm -def wat2wasm(code_wat: str) -> bytes: - """ - Converts the given WebAssembly Assembly code into WebAssembly Binary - """ - path = os.environ.get('WAT2WASM', 'wat2wasm') - - with tempfile.NamedTemporaryFile('w+t') as input_fp: - input_fp.write(code_wat) - input_fp.flush() - - with tempfile.NamedTemporaryFile('w+b') as output_fp: - subprocess.run( - [ - path, - input_fp.name, - '-o', - output_fp.name, - ], - check=True, - ) - - output_fp.seek(0) - - return output_fp.read() - class RunnerBase: """ Base class @@ -90,7 +63,7 @@ class RunnerBase: """ Compiles the WebAssembly AST into WebAssembly Binary """ - self.wasm_bin = wat2wasm(self.wasm_asm) + self.wasm_bin = wasmer.wat2wasm(self.wasm_asm) def interpreter_setup(self) -> None: """ @@ -253,6 +226,56 @@ class RunnerWasmtime(RunnerBase): return func(self.store, *args) +class RunnerWasmer(RunnerBase): + """ + Implements a runner for wasmer + + See https://pypi.org/project/wasmer/ + """ + + # pylint: disable=E1101 + + store: wasmer.Store + module: wasmer.Module + instance: wasmer.Instance + + def interpreter_setup(self) -> None: + self.store = wasmer.Store() + + def interpreter_load(self) -> None: + self.module = wasmer.Module(self.store, self.wasm_bin) + self.instance = wasmer.Instance(self.module) + + def interpreter_write_memory(self, offset: int, data: Iterable[int]) -> None: + exports = self.instance.exports + memory = getattr(exports, 'memory') + assert isinstance(memory, wasmer.Memory) + view = memory.uint8_view(offset) + + for idx, byt in enumerate(data): + view[idx] = byt + + def interpreter_read_memory(self, offset: int, length: int) -> bytes: + exports = self.instance.exports + memory = getattr(exports, 'memory') + assert isinstance(memory, wasmer.Memory) + view = memory.uint8_view(offset) + return bytes(view[offset:length]) + + def interpreter_dump_memory(self, textio: TextIO) -> None: + exports = self.instance.exports + memory = getattr(exports, 'memory') + assert isinstance(memory, wasmer.Memory) + view = memory.uint8_view() + + _dump_memory(textio, view) # type: ignore + + def call(self, function: str, *args: Any) -> Any: + exports = self.instance.exports + func = getattr(exports, function) + + return func(*args) + def _dump_memory(textio: TextIO, mem: bytes) -> None: line_width = 16 From a13713d7094d6607bd4ca82250250cd111521faf Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Tue, 9 Aug 2022 19:04:10 +0200 Subject: [PATCH 59/73] Cleanup to helpers, making use of runners --- TODO.md | 2 +- stubs/wasmer.pyi | 39 ++++ tests/integration/helpers.py | 237 ++++++------------------- tests/integration/runners.py | 30 +++- tests/integration/test_simple.py | 13 -- tests/integration/test_stdlib_alloc.py | 16 +- 6 files changed, 123 insertions(+), 214 deletions(-) create mode 100644 stubs/wasmer.pyi diff --git a/TODO.md b/TODO.md index afe6948..048c4ca 100644 --- a/TODO.md +++ b/TODO.md @@ -2,5 +2,5 @@ - Rename ourlang.Struct to OurTypeStruct - Maybe rename this to a types module? -- Remove references to log_int32_list - Fix naming in Suite() calls +- Replace ___new_reference___ by stdlib.alloc.__alloc__ diff --git a/stubs/wasmer.pyi b/stubs/wasmer.pyi new file mode 100644 index 0000000..535fa99 --- /dev/null +++ b/stubs/wasmer.pyi @@ -0,0 +1,39 @@ +from typing import Any, Dict, Callable, Union + +def wat2wasm(inp: str) -> bytes: + ... + +class Store: + ... + +class Function: + def __init__(self, store: Store, func: Callable[[Any], Any]) -> None: + ... + +class Module: + def __init__(self, store: Store, wasm: bytes) -> None: + ... + +class Uint8Array: + def __getitem__(self, index: Union[int, slice]) -> int: + ... + + def __setitem__(self, idx: int, value: int) -> None: + ... + +class Memory: + def uint8_view(self, offset: int = 0) -> Uint8Array: + ... + +class Exports: + ... + +class ImportObject: + def register(self, region: str, values: Dict[str, Function]) -> None: + ... + +class Instance: + exports: Exports + + def __init__(self, module: Module, imports: ImportObject) -> None: + ... diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index cfad8ed..f62004d 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -18,6 +18,8 @@ from phasm.codestyle import phasm_render from phasm.compiler import phasm_compile from phasm.parser import phasm_parse +from . import runners + DASHES = '-' * 16 def wat2wasm(code_wat): @@ -44,20 +46,14 @@ def wat2wasm(code_wat): class SuiteResult: def __init__(self): - self.log_int32_list = [] self.returned_value = None - def callback_log_int32(self, store, value): - del store # auto passed by pywasm - - self.log_int32_list.append(value) - - def make_imports(self): - return { - 'console': { - 'logInt32': self.callback_log_int32, - } - } +RUNNER_CLASS_MAP = { + 'pywasm': runners.RunnerPywasm, + 'pywasm3': runners.RunnerPywasm3, + 'wasmtime': runners.RunnerWasmtime, + 'wasmer': runners.RunnerWasmer, +} class Suite: """ @@ -73,189 +69,60 @@ class Suite: Returned is an object with the results set """ - phasm_module = phasm_parse(self.code_py) + class_ = RUNNER_CLASS_MAP[runtime] + + runner = class_(self.code_py) + + runner.parse() + runner.compile_ast() + runner.compile_wat() + runner.compile_wasm() + runner.interpreter_setup() + runner.interpreter_load(imports) + + write_header(sys.stderr, 'Phasm') + runner.dump_phasm_code(sys.stderr) + write_header(sys.stderr, 'Assembly') + runner.dump_wasm_wat(sys.stderr) # Check if code formatting works - assert self.code_py == '\n' + phasm_render(phasm_module) # \n for formatting in tests + assert self.code_py == '\n' + phasm_render(runner.phasm_ast) # \n for formatting in tests - # Compile - wasm_module = phasm_compile(phasm_module) + # runner.call('stdlib.alloc.__init__') - # Render as WebAssembly text - code_wat = wasm_module.to_wat() + wasm_args = [] + if args: + write_header(sys.stderr, 'Memory (pre alloc)') + runner.interpreter_dump_memory(sys.stderr) - sys.stderr.write(f'{DASHES} Assembly {DASHES}\n') + for arg in args: + if isinstance(arg, (int, float, )): + wasm_args.append(arg) + continue - _write_numbered_lines(code_wat) + if isinstance(arg, bytes): + # TODO: Implement and use the bytes constructor function + # TODO: call upon stdlib.alloc.__init__ and stdlib.alloc.__alloc__ + adr = runner.call('___new_reference___', len(arg) + 4) + sys.stderr.write(f'Allocation 0x{adr:08x} {repr(arg)}\n') - # Compile to assembly code - code_wasm = wat2wasm(code_wat) + runner.interpreter_write_memory(adr, len(arg).to_bytes(4, byteorder='little')) + runner.interpreter_write_memory(adr + 4, arg) + wasm_args.append(adr) + continue - # Run assembly code - if 'pywasm' == runtime: - return _run_pywasm(code_wasm, args) - if 'pywasm3' == runtime: - return _run_pywasm3(code_wasm, args) - if 'wasmtime' == runtime: - return _run_wasmtime(code_wasm, args) - if 'wasmer' == runtime: - return _run_wasmer(code_wasm, args, imports) + raise NotImplementedError(arg) - raise Exception(f'Invalid runtime: {runtime}') + write_header(sys.stderr, 'Memory (pre run)') + runner.interpreter_dump_memory(sys.stderr) -def _run_pywasm(code_wasm, args): - # https://pypi.org/project/pywasm/ - result = SuiteResult() + result = SuiteResult() + result.returned_value = runner.call('testEntry', *wasm_args) - module = pywasm.binary.Module.from_reader(io.BytesIO(code_wasm)) + write_header(sys.stderr, 'Memory (post run)') + runner.interpreter_dump_memory(sys.stderr) - runtime = pywasm.Runtime(module, result.make_imports(), {}) + return result - def set_byte(idx, byt): - runtime.store.mems[0].data[idx] = byt - - args = _convert_bytes_arguments( - args, - lambda x: runtime.exec('___new_reference___', [x]), - set_byte - ) - - sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') - _dump_memory(runtime.store.mems[0].data) - - result.returned_value = runtime.exec('testEntry', args) - - sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n') - _dump_memory(runtime.store.mems[0].data) - - return result - -def _run_pywasm3(code_wasm, args): - # https://pypi.org/project/pywasm3/ - result = SuiteResult() - - env = wasm3.Environment() - - mod = env.parse_module(code_wasm) - - rtime = env.new_runtime(1024 * 1024) - rtime.load(mod) - - def set_byte(idx, byt): - rtime.get_memory(0)[idx] = byt - - args = _convert_bytes_arguments( - args, - rtime.find_function('___new_reference___'), - set_byte - ) - - sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') - _dump_memory(rtime.get_memory(0)) - - result.returned_value = rtime.find_function('testEntry')(*args) - - sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n') - _dump_memory(rtime.get_memory(0)) - - return result - -def _run_wasmtime(code_wasm, args): - # https://pypi.org/project/wasmtime/ - result = SuiteResult() - - store = wasmtime.Store() - - module = wasmtime.Module(store.engine, code_wasm) - - instance = wasmtime.Instance(store, module, []) - - sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') - sys.stderr.write('\n') - - result.returned_value = instance.exports(store)['testEntry'](store, *args) - - sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n') - sys.stderr.write('\n') - - return result - -def _run_wasmer(code_wasm, args, imports): - # https://pypi.org/project/wasmer/ - result = SuiteResult() - - store = wasmer.Store(wasmer.engine.JIT(wasmer_compiler_cranelift.Compiler)) - - import_object = wasmer.ImportObject() - import_object.register('imports', { - k: wasmer.Function(store, v) - for k, v in (imports or {}).items() - }) - - # Let's compile the module to be able to execute it! - module = wasmer.Module(store, code_wasm) - - # Now the module is compiled, we can instantiate it. - instance = wasmer.Instance(module, import_object) - - sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') - sys.stderr.write('\n') - - # Call the exported `sum` function. - result.returned_value = instance.exports.testEntry(*args) - - sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n') - sys.stderr.write('\n') - - return result - -def _convert_bytes_arguments(args, new_reference, set_byte): - result = [] - for arg in args: - if not isinstance(arg, bytes): - result.append(arg) - continue - - # TODO: Implement and use the bytes constructor function - offset = new_reference(len(arg) + 4) - result.append(offset) - - # Store the length prefix - for idx, byt in enumerate(len(arg).to_bytes(4, byteorder='little')): - set_byte(offset + idx, byt) - - # Store the actual bytes - for idx, byt in enumerate(arg): - set_byte(offset + 4 + idx, byt) - - return result - -def _dump_memory(mem): - line_width = 16 - - prev_line = None - skip = False - for idx in range(0, len(mem), line_width): - line = '' - for idx2 in range(0, line_width): - line += f'{mem[idx + idx2]:02X}' - if idx2 % 2 == 1: - line += ' ' - - if prev_line == line: - if not skip: - sys.stderr.write('**\n') - skip = True - else: - sys.stderr.write(f'{idx:08x} {line}\n') - - prev_line = line - -def _write_numbered_lines(text: str) -> None: - line_list = text.split('\n') - line_no_width = len(str(len(line_list))) - for line_no, line_txt in enumerate(line_list): - sys.stderr.write('{} {}\n'.format( - str(line_no + 1).zfill(line_no_width), - line_txt, - )) +def write_header(textio, msg: str) -> None: + textio.write(f'{DASHES} {msg.ljust(16)} {DASHES}\n') diff --git a/tests/integration/runners.py b/tests/integration/runners.py index bb027f5..fd3a53e 100644 --- a/tests/integration/runners.py +++ b/tests/integration/runners.py @@ -1,7 +1,7 @@ """ Runners to help run WebAssembly code on various interpreters """ -from typing import Any, Iterable, TextIO +from typing import Any, Callable, Dict, Iterable, Optional, TextIO import ctypes import io @@ -71,7 +71,7 @@ class RunnerBase: """ raise NotImplementedError - def interpreter_load(self) -> None: + def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None: """ Loads the code into the interpreter """ @@ -114,7 +114,10 @@ class RunnerPywasm(RunnerBase): # Nothing to set up pass - def interpreter_load(self) -> None: + def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None: + if imports is not None: + raise NotImplementedError + bytesio = io.BytesIO(self.wasm_bin) self.module = pywasm.binary.Module.from_reader(bytesio) self.runtime = pywasm.Runtime(self.module, {}, None) @@ -146,7 +149,10 @@ class RunnerPywasm3(RunnerBase): self.env = wasm3.Environment() self.rtime = self.env.new_runtime(1024 * 1024) - def interpreter_load(self) -> None: + def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None: + if imports is not None: + raise NotImplementedError + self.mod = self.env.parse_module(self.wasm_bin) self.rtime.load(self.mod) @@ -179,7 +185,10 @@ class RunnerWasmtime(RunnerBase): def interpreter_setup(self) -> None: self.store = wasmtime.Store() - def interpreter_load(self) -> None: + def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None: + if imports is not None: + raise NotImplementedError + self.module = wasmtime.Module(self.store.engine, self.wasm_bin) self.instance = wasmtime.Instance(self.store, self.module, []) @@ -242,9 +251,16 @@ class RunnerWasmer(RunnerBase): def interpreter_setup(self) -> None: self.store = wasmer.Store() - def interpreter_load(self) -> None: + def interpreter_load(self, imports: Optional[Dict[str, Callable[[Any], Any]]] = None) -> None: + import_object = wasmer.ImportObject() + if imports: + import_object.register('imports', { + k: wasmer.Function(self.store, v) + for k, v in (imports or {}).items() + }) + self.module = wasmer.Module(self.store, self.wasm_bin) - self.instance = wasmer.Instance(self.module) + self.instance = wasmer.Instance(self.module, import_object) def interpreter_write_memory(self, offset: int, data: Iterable[int]) -> None: exports = self.instance.exports diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index b2d0bb2..2ea2f8d 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -114,7 +114,6 @@ def testEntry(a: i32) -> i64: result = Suite(code_py).run_code(125) assert 125 == result.returned_value - assert [] == result.log_int32_list @pytest.mark.integration_test @pytest.mark.skip('Do we want it to work like this?') @@ -128,7 +127,6 @@ def testEntry(a: i32, b: i64) -> i64: result = Suite(code_py).run_code(125, 100) assert 225 == result.returned_value - assert [] == result.log_int32_list @pytest.mark.integration_test @pytest.mark.skip('Do we want it to work like this?') @@ -142,7 +140,6 @@ def testEntry(a: f32) -> f64: result = Suite(code_py).run_code(125.5) assert 125.5 == result.returned_value - assert [] == result.log_int32_list @pytest.mark.integration_test @pytest.mark.skip('Do we want it to work like this?') @@ -156,7 +153,6 @@ def testEntry(a: f32, b: f64) -> f64: result = Suite(code_py).run_code(125.5, 100.25) assert 225.75 == result.returned_value - assert [] == result.log_int32_list @pytest.mark.integration_test @pytest.mark.skip('TODO') @@ -170,7 +166,6 @@ def testEntry() -> i32: result = Suite(code_py).run_code() assert 523 == result.returned_value - assert [] == result.log_int32_list @pytest.mark.integration_test @pytest.mark.skip('TODO') @@ -184,7 +179,6 @@ def testEntry() -> i32: result = Suite(code_py).run_code() assert -19 == result.returned_value - assert [] == result.log_int32_list @pytest.mark.integration_test @pytest.mark.parametrize('inp', [9, 10, 11, 12]) @@ -268,7 +262,6 @@ def testEntry() -> i32: result = Suite(code_py).run_code() assert 13 == result.returned_value - assert [] == result.log_int32_list @pytest.mark.integration_test def test_call_post_defined(): @@ -284,7 +277,6 @@ def helper(left: i32, right: i32) -> i32: result = Suite(code_py).run_code() assert 7 == result.returned_value - assert [] == result.log_int32_list @pytest.mark.integration_test @pytest.mark.parametrize('type_', COMPLETE_SIMPLE_TYPES) @@ -317,7 +309,6 @@ def testEntry() -> i32: result = Suite(code_py).run_code() assert 8947 == result.returned_value - assert [] == result.log_int32_list @pytest.mark.integration_test @pytest.mark.parametrize('type_', TYPE_MAP.keys()) @@ -337,7 +328,6 @@ def helper(cv: CheckedValue) -> {type_}: result = Suite(code_py).run_code() assert 23 == result.returned_value - assert [] == result.log_int32_list @pytest.mark.integration_test def test_struct_1(): @@ -358,7 +348,6 @@ def helper(shape: Rectangle) -> i32: result = Suite(code_py).run_code() assert 252 == result.returned_value - assert [] == result.log_int32_list @pytest.mark.integration_test def test_struct_2(): @@ -379,7 +368,6 @@ def helper(shape1: Rectangle, shape2: Rectangle) -> i32: result = Suite(code_py).run_code() assert 545 == result.returned_value - assert [] == result.log_int32_list @pytest.mark.integration_test @pytest.mark.parametrize('type_', COMPLETE_SIMPLE_TYPES) @@ -412,7 +400,6 @@ def helper(v: (f32, f32, f32, )) -> f32: result = Suite(code_py).run_code() assert 3.74 < result.returned_value < 3.75 - assert [] == result.log_int32_list @pytest.mark.integration_test def test_bytes_address(): diff --git a/tests/integration/test_stdlib_alloc.py b/tests/integration/test_stdlib_alloc.py index 7e960d7..a9faf35 100644 --- a/tests/integration/test_stdlib_alloc.py +++ b/tests/integration/test_stdlib_alloc.py @@ -2,7 +2,7 @@ import sys import pytest -from .helpers import DASHES +from .helpers import write_header from .runners import RunnerPywasm3 as Runner def setup_interpreter(phash_code: str) -> Runner: @@ -15,9 +15,9 @@ def setup_interpreter(phash_code: str) -> Runner: runner.interpreter_setup() runner.interpreter_load() - sys.stderr.write(f'{DASHES} Phasm {DASHES}\n') + write_header(sys.stderr, 'Phasm') runner.dump_phasm_code(sys.stderr) - sys.stderr.write(f'{DASHES} Assembly {DASHES}\n') + write_header(sys.stderr, 'Assembly') runner.dump_wasm_wat(sys.stderr) return runner @@ -35,12 +35,12 @@ def testEntry() -> u8: # Garbage in the memory so we can test for it runner.interpreter_write_memory(0, range(128)) - sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') + write_header(sys.stderr, 'Memory (pre run)') runner.interpreter_dump_memory(sys.stderr) runner.call('stdlib.alloc.__init__') - sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n') + write_header(sys.stderr, 'Memory (post run)') runner.interpreter_dump_memory(sys.stderr) assert ( @@ -61,7 +61,7 @@ def testEntry() -> u8: runner = setup_interpreter(code_py) - sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') + write_header(sys.stderr, 'Memory (pre run)') runner.interpreter_dump_memory(sys.stderr) with pytest.raises(Exception, match='unreachable'): @@ -77,7 +77,7 @@ def testEntry() -> u8: runner = setup_interpreter(code_py) - sys.stderr.write(f'{DASHES} Memory (pre run) {DASHES}\n') + write_header(sys.stderr, 'Memory (pre run)') runner.interpreter_dump_memory(sys.stderr) runner.call('stdlib.alloc.__init__') @@ -85,7 +85,7 @@ def testEntry() -> u8: offset1 = runner.call('stdlib.alloc.__alloc__', 32) offset2 = runner.call('stdlib.alloc.__alloc__', 32) - sys.stderr.write(f'{DASHES} Memory (post run) {DASHES}\n') + write_header(sys.stderr, 'Memory (post run)') runner.interpreter_dump_memory(sys.stderr) assert ( From a0d575f61fd45464bc9086850770e0a49aa16a59 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Tue, 9 Aug 2022 20:21:59 +0200 Subject: [PATCH 60/73] Implements __alloc_bytes__, uses it in the buffer example Also, updated todo, remove broken code from buffer example --- Makefile | 2 +- TODO.md | 5 ++--- examples/buffer.html | 14 ++++++-------- examples/buffer.py | 4 ---- phasm/compiler.py | 3 ++- phasm/stdlib/types.py | 32 ++++++++++++++++++++++++++++++++ 6 files changed, 43 insertions(+), 17 deletions(-) create mode 100644 phasm/stdlib/types.py diff --git a/Makefile b/Makefile index 09bbb51..8f4d045 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ WABT_DIR := /home/johan/Sources/github.com/WebAssembly/wabt WAT2WASM := $(WABT_DIR)/bin/wat2wasm WASM2C := $(WABT_DIR)/bin/wasm2c -%.wat: %.py phasm/*.py venv/.done +%.wat: %.py $(shell find phasm -name '*.py') venv/.done venv/bin/python -m phasm $< $@ %.wat.html: %.wat diff --git a/TODO.md b/TODO.md index 048c4ca..2775e07 100644 --- a/TODO.md +++ b/TODO.md @@ -1,6 +1,5 @@ # TODO -- Rename ourlang.Struct to OurTypeStruct - - Maybe rename this to a types module? -- Fix naming in Suite() calls +- Implement foldl for bytes - Replace ___new_reference___ by stdlib.alloc.__alloc__ +- Implement a trace() builtin for debugging diff --git a/examples/buffer.html b/examples/buffer.html index d0b14ca..e5d7c19 100644 --- a/examples/buffer.html +++ b/examples/buffer.html @@ -30,22 +30,20 @@ WebAssembly.instantiateStreaming(fetch('buffer.wasm'), importObject) // Allocate room within the memory of the WebAssembly class let size = 8; - // TODO: Use bytes constructor - let offset = app.instance.exports.___new_reference___(size); - new Uint32Array(app.instance.exports.memory.buffer, offset, 1)[0] = size; + stdlib_alloc___init__ = app.instance.exports['stdlib.alloc.__init__'] + stdlib_alloc___init__() + + stdlib_types___alloc_bytes__ = app.instance.exports['stdlib.types.__alloc_bytes__'] + + let offset = stdlib_types___alloc_bytes__(size) var i8arr = new Uint8Array(app.instance.exports.memory.buffer, offset + 4, size); //Fill it - let sum = 0; for (var i = 0; i < size; i++) { i8arr[i] = i + 5; - sum += i8arr[i]; let from_wasm = app.instance.exports.index(offset, i); log('i8arr[' + i + '] = ' + from_wasm); } - - log('expected result = ' + sum); - log('actual result = ' + app.instance.exports.sum(offset)); }); diff --git a/examples/buffer.py b/examples/buffer.py index 74eb023..96944fd 100644 --- a/examples/buffer.py +++ b/examples/buffer.py @@ -1,7 +1,3 @@ -@exported -def sum() -> i32: # TODO: Should be [i32] -> i32 - return 0 # TODO: Implement me - @exported def index(inp: bytes, idx: i32) -> i32: return inp[idx] diff --git a/phasm/compiler.py b/phasm/compiler.py index 6c9ce98..eb02672 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -6,9 +6,9 @@ from typing import Generator from . import ourlang from . import typing from . import wasm -from . import wasmeasy from .stdlib import alloc as stdlib_alloc +from .stdlib import types as stdlib_types from .wasmeasy import i32, i64 Statements = Generator[wasm.Statement, None, None] @@ -380,6 +380,7 @@ def module(inp: ourlang.Module) -> wasm.Module: stdlib_alloc.__init__, stdlib_alloc.__find_free_block__, stdlib_alloc.__alloc__, + stdlib_types.__alloc_bytes__, _generate____access_bytes_index___(inp), ] + [ function(x) diff --git a/phasm/stdlib/types.py b/phasm/stdlib/types.py new file mode 100644 index 0000000..f815ead --- /dev/null +++ b/phasm/stdlib/types.py @@ -0,0 +1,32 @@ +""" +stdlib: Standard types that are not wasm primitives +""" +from phasm.wasmgenerator import Generator, VarType_i32 as i32, func_wrapper + +from phasm.stdlib import alloc + +@func_wrapper() +def __alloc_bytes__(g: Generator, length: i32) -> i32: + """ + Allocates room for a bytes instance, but does not write + anything to the allocated memory + """ + result = i32('result') + + # Allocate the length of the byte string, as well + # as 4 bytes for a length header + g.local.get(length) + g.i32.const(4) + g.i32.add() + g.call(alloc.__alloc__) + + # Store the address in a variable so we can use it up + # for writing the length header + g.local.tee(result) + g.local.get(length) + g.i32.store() + + # Get the address back from the variable as return + g.local.get(result) + + return i32('return') # To satisfy mypy From 4881cb6d4b47810d90fd8b2a8c8536b2a4380f2e Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Tue, 9 Aug 2022 20:34:03 +0200 Subject: [PATCH 61/73] Moved ___access_bytes_index___ to the right place Also, added length test to buffer example --- examples/buffer.html | 2 ++ examples/buffer.py | 4 ++++ phasm/compiler.py | 36 ++---------------------------------- phasm/stdlib/alloc.py | 2 +- phasm/stdlib/types.py | 34 ++++++++++++++++++++++++++++++++++ phasm/wasmgenerator.py | 2 ++ 6 files changed, 45 insertions(+), 35 deletions(-) diff --git a/examples/buffer.html b/examples/buffer.html index e5d7c19..3264f9b 100644 --- a/examples/buffer.html +++ b/examples/buffer.html @@ -38,6 +38,8 @@ WebAssembly.instantiateStreaming(fetch('buffer.wasm'), importObject) let offset = stdlib_types___alloc_bytes__(size) var i8arr = new Uint8Array(app.instance.exports.memory.buffer, offset + 4, size); + log('len(i8arr) = ' + app.instance.exports.length(offset)); + //Fill it for (var i = 0; i < size; i++) { i8arr[i] = i + 5; diff --git a/examples/buffer.py b/examples/buffer.py index 96944fd..51096c6 100644 --- a/examples/buffer.py +++ b/examples/buffer.py @@ -1,3 +1,7 @@ @exported def index(inp: bytes, idx: i32) -> i32: return inp[idx] + +@exported +def length(inp: bytes) -> i32: + return len(inp) diff --git a/phasm/compiler.py b/phasm/compiler.py index eb02672..58186de 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -231,7 +231,7 @@ def expression(inp: ourlang.Expression) -> Statements: yield from expression(inp.varref) yield from expression(inp.index) - yield wasm.Statement('call', '$___access_bytes_index___') + yield wasm.Statement('call', '$stdlib.types.__subscript_bytes__') return if isinstance(inp, ourlang.AccessStructMember): @@ -381,7 +381,7 @@ def module(inp: ourlang.Module) -> wasm.Module: stdlib_alloc.__find_free_block__, stdlib_alloc.__alloc__, stdlib_types.__alloc_bytes__, - _generate____access_bytes_index___(inp), + stdlib_types.__subscript_bytes__, ] + [ function(x) for x in inp.functions.values() @@ -413,38 +413,6 @@ def _generate____new_reference___(mod: ourlang.Module) -> wasm.Function: ], ) -def _generate____access_bytes_index___(mod: ourlang.Module) -> wasm.Function: - return wasm.Function( - '___access_bytes_index___', - None, - [ - ('byt', type_(mod.types['i32']), ), - ('ofs', type_(mod.types['i32']), ), - ], - [ - ], - type_(mod.types['i32']), - [ - wasm.Statement('local.get', '$ofs'), - wasm.Statement('local.get', '$byt'), - i32.load(), - wasm.Statement('i32.lt_u'), - - wasm.Statement('if', comment='$ofs < len($byt)'), - wasm.Statement('local.get', '$byt'), - wasm.Statement('i32.const', '4', comment='Leading size field'), - wasm.Statement('i32.add'), - wasm.Statement('local.get', '$ofs'), - wasm.Statement('i32.add'), - wasm.Statement('i32.load8_u', comment='Within bounds'), - wasm.Statement('return'), - wasm.Statement('end'), - - wasm.Statement('i32.const', str(0), comment='Out of bounds'), - wasm.Statement('return'), - ], - ) - def _generate_tuple_constructor(inp: ourlang.TupleConstructor) -> Statements: yield wasm.Statement('i32.const', str(inp.tuple.alloc_size())) yield wasm.Statement('call', '$___new_reference___') diff --git a/phasm/stdlib/alloc.py b/phasm/stdlib/alloc.py index 73608ff..a803545 100644 --- a/phasm/stdlib/alloc.py +++ b/phasm/stdlib/alloc.py @@ -113,7 +113,7 @@ def __alloc__(g: Generator, alloc_size: i32) -> i32: g.local.get(alloc_size) g.i32.store() - # Return address of the allocated bytes + # Return address of the allocated memory, skipping the allocator header g.local.get(result) g.i32.const(4) # Header size g.i32.add() diff --git a/phasm/stdlib/types.py b/phasm/stdlib/types.py index f815ead..b902642 100644 --- a/phasm/stdlib/types.py +++ b/phasm/stdlib/types.py @@ -30,3 +30,37 @@ def __alloc_bytes__(g: Generator, length: i32) -> i32: g.local.get(result) return i32('return') # To satisfy mypy + +@func_wrapper() +def __subscript_bytes__(g: Generator, adr: i32, ofs: i32) -> i32: + """ + Returns an index from a bytes value + + If ofs is more than the length of the bytes, this + function returns 0, following the 'no undefined behaviour' + philosophy. + + adr i32 The pointer for the allocated bytes + ofs i32 The offset within the allocated bytes + """ + g.local.get(ofs) + g.local.get(adr) + g.i32.load() + g.i32.lt_u() + + with g.if_(): + # The offset is less than the length + + g.local.get(adr) + g.i32.const(4) # Bytes header + g.i32.add() + g.local.get(ofs) + g.i32.add() + g.i32.load8_u() + g.return_() + + # The offset is outside the allocated bytes + g.i32.const(0) + g.return_() + + return i32('return') # To satisfy mypy diff --git a/phasm/wasmgenerator.py b/phasm/wasmgenerator.py index 1c8015a..2647296 100644 --- a/phasm/wasmgenerator.py +++ b/phasm/wasmgenerator.py @@ -30,9 +30,11 @@ class Generator_i32: # irelop self.eq = functools.partial(self.generator.add, 'i32.eq') self.ne = functools.partial(self.generator.add, 'i32.ne') + self.lt_u = functools.partial(self.generator.add, 'i32.lt_u') # 2.4.4. Memory Instructions self.load = functools.partial(self.generator.add, 'i32.load') + self.load8_u = functools.partial(self.generator.add, 'i32.load8_u') self.store = functools.partial(self.generator.add, 'i32.store') def const(self, value: int, comment: Optional[str] = None) -> None: From 451a8e915870e24451cd2bcc1c564ac993f32236 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Tue, 9 Aug 2022 20:42:02 +0200 Subject: [PATCH 62/73] Removes the old ___new_reference___ allocator --- TODO.md | 1 - phasm/compiler.py | 28 ++-------------------------- phasm/wasm.py | 2 +- tests/integration/helpers.py | 7 ++----- 4 files changed, 5 insertions(+), 33 deletions(-) diff --git a/TODO.md b/TODO.md index 2775e07..28a862b 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,4 @@ # TODO - Implement foldl for bytes -- Replace ___new_reference___ by stdlib.alloc.__alloc__ - Implement a trace() builtin for debugging diff --git a/phasm/compiler.py b/phasm/compiler.py index 58186de..13cc851 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -376,7 +376,6 @@ def module(inp: ourlang.Module) -> wasm.Module: ] result.functions = [ - _generate____new_reference___(inp), # Old allocator stdlib_alloc.__init__, stdlib_alloc.__find_free_block__, stdlib_alloc.__alloc__, @@ -390,32 +389,9 @@ def module(inp: ourlang.Module) -> wasm.Module: return result -def _generate____new_reference___(mod: ourlang.Module) -> wasm.Function: - return wasm.Function( - '___new_reference___', - '___new_reference___', - [ - ('alloc_size', type_(mod.types['i32']), ), - ], - [ - ('result', type_(mod.types['i32']), ), - ], - type_(mod.types['i32']), - [ - i32.const(0), - i32.const(0), - i32.load(), - wasm.Statement('local.tee', '$result', comment='Address for this call'), - wasm.Statement('local.get', '$alloc_size'), - i32.add(), - i32.store(comment='Address for the next call'), - wasm.Statement('local.get', '$result'), - ], - ) - def _generate_tuple_constructor(inp: ourlang.TupleConstructor) -> Statements: yield wasm.Statement('i32.const', str(inp.tuple.alloc_size())) - yield wasm.Statement('call', '$___new_reference___') + yield wasm.Statement('call', '$stdlib.alloc.__alloc__') yield wasm.Statement('local.set', '$___new_reference___addr') @@ -434,7 +410,7 @@ def _generate_tuple_constructor(inp: ourlang.TupleConstructor) -> Statements: def _generate_struct_constructor(inp: ourlang.StructConstructor) -> Statements: yield wasm.Statement('i32.const', str(inp.struct.alloc_size())) - yield wasm.Statement('call', '$___new_reference___') + yield wasm.Statement('call', '$stdlib.alloc.__alloc__') yield wasm.Statement('local.set', '$___new_reference___addr') diff --git a/phasm/wasm.py b/phasm/wasm.py index 5fa2506..7c5a982 100644 --- a/phasm/wasm.py +++ b/phasm/wasm.py @@ -186,7 +186,7 @@ class Module(WatSerializable): def __init__(self) -> None: self.imports: List[Import] = [] self.functions: List[Function] = [] - self.memory = ModuleMemory(b'\x04') # For ___new_reference___ + self.memory = ModuleMemory() def to_wat(self) -> str: """ diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index f62004d..2e6b706 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -88,7 +88,7 @@ class Suite: # Check if code formatting works assert self.code_py == '\n' + phasm_render(runner.phasm_ast) # \n for formatting in tests - # runner.call('stdlib.alloc.__init__') + runner.call('stdlib.alloc.__init__') wasm_args = [] if args: @@ -101,12 +101,9 @@ class Suite: continue if isinstance(arg, bytes): - # TODO: Implement and use the bytes constructor function - # TODO: call upon stdlib.alloc.__init__ and stdlib.alloc.__alloc__ - adr = runner.call('___new_reference___', len(arg) + 4) + adr = runner.call('stdlib.types.__alloc_bytes__', len(arg)) sys.stderr.write(f'Allocation 0x{adr:08x} {repr(arg)}\n') - runner.interpreter_write_memory(adr, len(arg).to_bytes(4, byteorder='little')) runner.interpreter_write_memory(adr + 4, arg) wasm_args.append(adr) continue From 0cf8a246fe0074124b2035b05f399e21c2e62994 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Wed, 10 Aug 2022 20:51:01 +0200 Subject: [PATCH 63/73] Reworked compiler so it uses WasmGenerator Also, started work on foldl Also, added a missing FIXME --- TODO.md | 1 + phasm/compiler.py | 201 ++++++++++++++++--------------- phasm/ourlang.py | 33 +++++ phasm/parser.py | 35 +++++- phasm/typing.py | 2 +- phasm/wasmgenerator.py | 98 ++++++++++++--- tests/integration/test_simple.py | 4 +- 7 files changed, 256 insertions(+), 118 deletions(-) diff --git a/TODO.md b/TODO.md index 28a862b..b2ab013 100644 --- a/TODO.md +++ b/TODO.md @@ -2,3 +2,4 @@ - Implement foldl for bytes - Implement a trace() builtin for debugging +- Implement a proper type matching / checking system diff --git a/phasm/compiler.py b/phasm/compiler.py index 13cc851..e693601 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -1,17 +1,14 @@ """ This module contains the code to convert parsed Ourlang into WebAssembly code """ -from typing import Generator - +from . import codestyle from . import ourlang from . import typing from . import wasm from .stdlib import alloc as stdlib_alloc from .stdlib import types as stdlib_types -from .wasmeasy import i32, i64 - -Statements = Generator[wasm.Statement, None, None] +from .wasmgenerator import Generator as WasmGenerator LOAD_STORE_TYPE_MAP = { typing.TypeUInt8: 'i32', @@ -115,123 +112,123 @@ I64_OPERATOR_MAP = { '>=': 'ge_s', } -def expression(inp: ourlang.Expression) -> Statements: +def expression(wgn: WasmGenerator, inp: ourlang.Expression) -> None: """ Compile: Any expression """ if isinstance(inp, ourlang.ConstantUInt8): - yield i32.const(inp.value) + wgn.i32.const(inp.value) return if isinstance(inp, ourlang.ConstantUInt32): - yield i32.const(inp.value) + wgn.i32.const(inp.value) return if isinstance(inp, ourlang.ConstantUInt64): - yield i64.const(inp.value) + wgn.i64.const(inp.value) return if isinstance(inp, ourlang.ConstantInt32): - yield i32.const(inp.value) + wgn.i32.const(inp.value) return if isinstance(inp, ourlang.ConstantInt64): - yield i64.const(inp.value) + wgn.i64.const(inp.value) return if isinstance(inp, ourlang.ConstantFloat32): - yield wasm.Statement('f32.const', str(inp.value)) + wgn.f32.const(inp.value) return if isinstance(inp, ourlang.ConstantFloat64): - yield wasm.Statement('f64.const', str(inp.value)) + wgn.f64.const(inp.value) return if isinstance(inp, ourlang.VariableReference): - yield wasm.Statement('local.get', '${}'.format(inp.name)) + wgn.add_statement('local.get', '${}'.format(inp.name)) return if isinstance(inp, ourlang.BinaryOp): - yield from expression(inp.left) - yield from expression(inp.right) + expression(wgn, inp.left) + expression(wgn, inp.right) if isinstance(inp.type, typing.TypeUInt8): if operator := U8_OPERATOR_MAP.get(inp.operator, None): - yield wasm.Statement(f'i32.{operator}') + wgn.add_statement(f'i32.{operator}') return if isinstance(inp.type, typing.TypeUInt32): if operator := OPERATOR_MAP.get(inp.operator, None): - yield wasm.Statement(f'i32.{operator}') + wgn.add_statement(f'i32.{operator}') return if operator := U32_OPERATOR_MAP.get(inp.operator, None): - yield wasm.Statement(f'i32.{operator}') + wgn.add_statement(f'i32.{operator}') return if isinstance(inp.type, typing.TypeUInt64): if operator := OPERATOR_MAP.get(inp.operator, None): - yield wasm.Statement(f'i64.{operator}') + wgn.add_statement(f'i64.{operator}') return if operator := U64_OPERATOR_MAP.get(inp.operator, None): - yield wasm.Statement(f'i64.{operator}') + wgn.add_statement(f'i64.{operator}') return if isinstance(inp.type, typing.TypeInt32): if operator := OPERATOR_MAP.get(inp.operator, None): - yield wasm.Statement(f'i32.{operator}') + wgn.add_statement(f'i32.{operator}') return if operator := I32_OPERATOR_MAP.get(inp.operator, None): - yield wasm.Statement(f'i32.{operator}') + wgn.add_statement(f'i32.{operator}') return if isinstance(inp.type, typing.TypeInt64): if operator := OPERATOR_MAP.get(inp.operator, None): - yield wasm.Statement(f'i64.{operator}') + wgn.add_statement(f'i64.{operator}') return if operator := I64_OPERATOR_MAP.get(inp.operator, None): - yield wasm.Statement(f'i64.{operator}') + wgn.add_statement(f'i64.{operator}') return if isinstance(inp.type, typing.TypeFloat32): if operator := OPERATOR_MAP.get(inp.operator, None): - yield wasm.Statement(f'f32.{operator}') + wgn.add_statement(f'f32.{operator}') return if isinstance(inp.type, typing.TypeFloat64): if operator := OPERATOR_MAP.get(inp.operator, None): - yield wasm.Statement(f'f64.{operator}') + wgn.add_statement(f'f64.{operator}') return raise NotImplementedError(expression, inp.type, inp.operator) if isinstance(inp, ourlang.UnaryOp): - yield from expression(inp.right) + expression(wgn, inp.right) if isinstance(inp.type, typing.TypeFloat32): if inp.operator in ourlang.WEBASSEMBLY_BUILDIN_FLOAT_OPS: - yield wasm.Statement(f'f32.{inp.operator}') + wgn.add_statement(f'f32.{inp.operator}') return if isinstance(inp.type, typing.TypeFloat64): if inp.operator in ourlang.WEBASSEMBLY_BUILDIN_FLOAT_OPS: - yield wasm.Statement(f'f64.{inp.operator}') + wgn.add_statement(f'f64.{inp.operator}') return if isinstance(inp.type, typing.TypeInt32): if inp.operator == 'len': if isinstance(inp.right.type, typing.TypeBytes): - yield i32.load() + wgn.i32.load() return raise NotImplementedError(expression, inp.type, inp.operator) if isinstance(inp, ourlang.FunctionCall): for arg in inp.arguments: - yield from expression(arg) + expression(wgn, arg) - yield wasm.Statement('call', '${}'.format(inp.function.name)) + wgn.add_statement('call', '${}'.format(inp.function.name)) return if isinstance(inp, ourlang.AccessBytesIndex): if not isinstance(inp.type, typing.TypeUInt8): raise NotImplementedError(inp, inp.type) - yield from expression(inp.varref) - yield from expression(inp.index) - yield wasm.Statement('call', '$stdlib.types.__subscript_bytes__') + expression(wgn, inp.varref) + expression(wgn, inp.index) + wgn.call(stdlib_types.__subscript_bytes__) return if isinstance(inp, ourlang.AccessStructMember): @@ -241,8 +238,8 @@ def expression(inp: ourlang.Expression) -> Statements: # as members of struct or tuples raise NotImplementedError(expression, inp, inp.member) - yield from expression(inp.varref) - yield wasm.Statement(f'{mtyp}.load', 'offset=' + str(inp.member.offset)) + expression(wgn, inp.varref) + wgn.add_statement(f'{mtyp}.load', 'offset=' + str(inp.member.offset)) return if isinstance(inp, ourlang.AccessTupleMember): @@ -252,47 +249,62 @@ def expression(inp: ourlang.Expression) -> Statements: # as members of struct or tuples raise NotImplementedError(expression, inp, inp.member) - yield from expression(inp.varref) - yield wasm.Statement(f'{mtyp}.load', 'offset=' + str(inp.member.offset)) + expression(wgn, inp.varref) + wgn.add_statement(f'{mtyp}.load', 'offset=' + str(inp.member.offset)) + return + + if isinstance(inp, ourlang.Fold): + mtyp = LOAD_STORE_TYPE_MAP.get(inp.base.type.__class__) + if mtyp is None: + # In the future might extend this by having structs or tuples + # as members of struct or tuples + raise NotImplementedError(expression, inp, inp.base) + + if inp.iter.type.__class__.__name__ != 'TypeBytes': + raise NotImplementedError(expression, inp, inp.iter.type) + + tmp_var = wgn.temp_var_u8(f'fold_{codestyle.type_(inp.type)}') + expression(wgn, inp.base) + wgn.local.set(tmp_var) + + # FIXME: Do a loop + wgn.unreachable(comment='Not implemented yet') return raise NotImplementedError(expression, inp) -def statement_return(inp: ourlang.StatementReturn) -> Statements: +def statement_return(wgn: WasmGenerator, inp: ourlang.StatementReturn) -> None: """ Compile: Return statement """ - yield from expression(inp.value) - yield wasm.Statement('return') + expression(wgn, inp.value) + wgn.return_() -def statement_if(inp: ourlang.StatementIf) -> Statements: +def statement_if(wgn: WasmGenerator, inp: ourlang.StatementIf) -> None: """ Compile: If statement """ - yield from expression(inp.test) + expression(wgn, inp.test) + with wgn.if_(): + for stat in inp.statements: + statement(wgn, stat) - yield wasm.Statement('if') + if inp.else_statements: + raise NotImplementedError + # yield wasm.Statement('else') + # for stat in inp.else_statements: + # statement(wgn, stat) - for stat in inp.statements: - yield from statement(stat) - - if inp.else_statements: - yield wasm.Statement('else') - for stat in inp.else_statements: - yield from statement(stat) - - yield wasm.Statement('end') - -def statement(inp: ourlang.Statement) -> Statements: +def statement(wgn: WasmGenerator, inp: ourlang.Statement) -> None: """ Compile: any statement """ if isinstance(inp, ourlang.StatementReturn): - yield from statement_return(inp) + statement_return(wgn, inp) return if isinstance(inp, ourlang.StatementIf): - yield from statement_if(inp) + statement_if(wgn, inp) return if isinstance(inp, ourlang.StatementPass): @@ -329,27 +341,15 @@ def function(inp: ourlang.Function) -> wasm.Function: """ assert not inp.imported + wgn = WasmGenerator() + if isinstance(inp, ourlang.TupleConstructor): - statements = [ - *_generate_tuple_constructor(inp) - ] - locals_ = [ - ('___new_reference___addr', wasm.WasmTypeInt32(), ), - ] + _generate_tuple_constructor(wgn, inp) elif isinstance(inp, ourlang.StructConstructor): - statements = [ - *_generate_struct_constructor(inp) - ] - locals_ = [ - ('___new_reference___addr', wasm.WasmTypeInt32(), ), - ] + _generate_struct_constructor(wgn, inp) else: - statements = [ - x - for y in inp.statements - for x in statement(y) - ] - locals_ = [] # FIXME: Implement function locals, if required + for stat in inp.statements: + statement(wgn, stat) return wasm.Function( inp.name, @@ -358,9 +358,12 @@ def function(inp: ourlang.Function) -> wasm.Function: function_argument(x) for x in inp.posonlyargs ], - locals_, + [ + (k, v.wasm_type(), ) + for k, v in wgn.locals.items() + ], type_(inp.returns), - statements + wgn.statements ) def module(inp: ourlang.Module) -> wasm.Module: @@ -389,12 +392,15 @@ def module(inp: ourlang.Module) -> wasm.Module: return result -def _generate_tuple_constructor(inp: ourlang.TupleConstructor) -> Statements: - yield wasm.Statement('i32.const', str(inp.tuple.alloc_size())) - yield wasm.Statement('call', '$stdlib.alloc.__alloc__') +def _generate_tuple_constructor(wgn: WasmGenerator, inp: ourlang.TupleConstructor) -> None: + tmp_var = wgn.temp_var_i32('tuple_adr') - yield wasm.Statement('local.set', '$___new_reference___addr') + # Allocated the required amounts of bytes in memory + wgn.i32.const(inp.tuple.alloc_size()) + wgn.call(stdlib_alloc.__alloc__) + wgn.local.set(tmp_var) + # Store each member individually for member in inp.tuple.members: mtyp = LOAD_STORE_TYPE_MAP.get(member.type.__class__) if mtyp is None: @@ -402,18 +408,22 @@ def _generate_tuple_constructor(inp: ourlang.TupleConstructor) -> Statements: # as members of struct or tuples raise NotImplementedError(expression, inp, member) - yield wasm.Statement('local.get', '$___new_reference___addr') - yield wasm.Statement('local.get', f'$arg{member.idx}') - yield wasm.Statement(f'{mtyp}.store', 'offset=' + str(member.offset)) + wgn.local.get(tmp_var) + wgn.add_statement('local.get', f'$arg{member.idx}') + wgn.add_statement(f'{mtyp}.store', 'offset=' + str(member.offset)) - yield wasm.Statement('local.get', '$___new_reference___addr') + # Return the allocated address + wgn.local.get(tmp_var) -def _generate_struct_constructor(inp: ourlang.StructConstructor) -> Statements: - yield wasm.Statement('i32.const', str(inp.struct.alloc_size())) - yield wasm.Statement('call', '$stdlib.alloc.__alloc__') +def _generate_struct_constructor(wgn: WasmGenerator, inp: ourlang.StructConstructor) -> None: + tmp_var = wgn.temp_var_i32('tuple_adr') - yield wasm.Statement('local.set', '$___new_reference___addr') + # Allocated the required amounts of bytes in memory + wgn.i32.const(inp.struct.alloc_size()) + wgn.call(stdlib_alloc.__alloc__) + wgn.local.set(tmp_var) + # Store each member individually for member in inp.struct.members: mtyp = LOAD_STORE_TYPE_MAP.get(member.type.__class__) if mtyp is None: @@ -421,8 +431,9 @@ def _generate_struct_constructor(inp: ourlang.StructConstructor) -> Statements: # as members of struct or tuples raise NotImplementedError(expression, inp, member) - yield wasm.Statement('local.get', '$___new_reference___addr') - yield wasm.Statement('local.get', f'${member.name}') - yield wasm.Statement(f'{mtyp}.store', 'offset=' + str(member.offset)) + wgn.local.get(tmp_var) + wgn.add_statement('local.get', f'${member.name}') + wgn.add_statement(f'{mtyp}.store', 'offset=' + str(member.offset)) - yield wasm.Statement('local.get', '$___new_reference___addr') + # Return the allocated address + wgn.local.get(tmp_var) diff --git a/phasm/ourlang.py b/phasm/ourlang.py index ef9e37a..4d01b16 100644 --- a/phasm/ourlang.py +++ b/phasm/ourlang.py @@ -3,6 +3,8 @@ Contains the syntax tree for ourlang """ from typing import Dict, List, Tuple +import enum + from typing_extensions import Final WEBASSEMBLY_BUILDIN_FLOAT_OPS: Final = ('abs', 'sqrt', 'ceil', 'floor', 'trunc', 'nearest', ) @@ -225,6 +227,37 @@ class AccessTupleMember(Expression): self.varref = varref self.member = member +class Fold(Expression): + """ + A (left or right) fold + """ + class Direction(enum.Enum): + """ + Which direction to fold in + """ + LEFT = 0 + RIGHT = 1 + + dir: Direction + func: 'Function' + base: Expression + iter: Expression + + def __init__( + self, + type_: TypeBase, + dir_: Direction, + func: 'Function', + base: Expression, + iter_: Expression, + ) -> None: + super().__init__(type_) + + self.dir = dir_ + self.func = func + self.base = base + self.iter = iter_ + class Statement: """ A statement within a function diff --git a/phasm/parser.py b/phasm/parser.py index 42d3bcc..d9fbbc1 100644 --- a/phasm/parser.py +++ b/phasm/parser.py @@ -38,6 +38,8 @@ from .ourlang import ( StructConstructor, TupleConstructor, UnaryOp, VariableReference, + Fold, + Statement, StatementIf, StatementPass, StatementReturn, ) @@ -348,7 +350,7 @@ class OurVisitor: raise NotImplementedError(f'{node} as expr in FunctionDef') - def visit_Module_FunctionDef_Call(self, module: Module, function: Function, our_locals: OurLocals, exp_type: TypeBase, node: ast.Call) -> Union[FunctionCall, UnaryOp]: + def visit_Module_FunctionDef_Call(self, module: Module, function: Function, our_locals: OurLocals, exp_type: TypeBase, node: ast.Call) -> Union[Fold, FunctionCall, UnaryOp]: if node.keywords: _raise_static_error(node, 'Keyword calling not supported') # Yet? @@ -386,6 +388,37 @@ class OurVisitor: 'len', self.visit_Module_FunctionDef_expr(module, function, our_locals, module.types['bytes'], node.args[0]), ) + elif node.func.id == 'foldl': + # TODO: This should a much more generic function! + # For development purposes, we're assuming you're doing a foldl(Callable[[u8, u8], u8], u8, bytes) + # In the future, we should probably infer the type of the second argument, + # and use it as expected types for the other u8s and the Iterable[u8] (i.e. bytes) + + if not isinstance(exp_type, TypeUInt8): + _raise_static_error(node, f'Cannot make {node.func.id} result in {exp_type} - not implemented yet') + + if 3 != len(node.args): + _raise_static_error(node, f'Function {node.func.id} requires 3 arguments but {len(node.args)} are given') + + t_u8 = module.types['u8'] + + # TODO: This is not generic + subnode = node.args[0] + if not isinstance(subnode, ast.Name): + raise NotImplementedError(f'Calling methods that are not a name {subnode}') + if not isinstance(subnode.ctx, ast.Load): + _raise_static_error(subnode, 'Must be load context') + if subnode.id not in module.functions: + _raise_static_error(subnode, 'Reference to undefined function') + func = module.functions[subnode.id] + + return Fold( + exp_type, + Fold.Direction.LEFT, + func, + self.visit_Module_FunctionDef_expr(module, function, our_locals, t_u8, node.args[1]), + self.visit_Module_FunctionDef_expr(module, function, our_locals, module.types['bytes'], node.args[2]), + ) else: if node.func.id not in module.functions: _raise_static_error(node, 'Call to undefined function') diff --git a/phasm/typing.py b/phasm/typing.py index 71a7549..0a29f5f 100644 --- a/phasm/typing.py +++ b/phasm/typing.py @@ -127,7 +127,7 @@ class TypeTuple(TypeBase): """ Generates an internal name for this tuple """ - mems = '@'.join('?' for x in self.members) + mems = '@'.join('?' for x in self.members) # FIXME: Should not be a questionmark assert ' ' not in mems, 'Not implement yet: subtuples' return f'tuple@{mems}' diff --git a/phasm/wasmgenerator.py b/phasm/wasmgenerator.py index 2647296..dc16589 100644 --- a/phasm/wasmgenerator.py +++ b/phasm/wasmgenerator.py @@ -16,29 +16,70 @@ class VarType_Base: self.name = name self.name_ref = f'${name}' +class VarType_u8(VarType_Base): + wasm_type = wasm.WasmTypeInt32 + class VarType_i32(VarType_Base): wasm_type = wasm.WasmTypeInt32 -class Generator_i32: - def __init__(self, generator: 'Generator') -> None: +class Generator_i32i64: + def __init__(self, prefix: str, generator: 'Generator') -> None: + self.prefix = prefix self.generator = generator # 2.4.1. Numeric Instructions # ibinop - self.add = functools.partial(self.generator.add, 'i32.add') + self.add = functools.partial(self.generator.add_statement, f'{prefix}.add') # irelop - self.eq = functools.partial(self.generator.add, 'i32.eq') - self.ne = functools.partial(self.generator.add, 'i32.ne') - self.lt_u = functools.partial(self.generator.add, 'i32.lt_u') + self.eq = functools.partial(self.generator.add_statement, f'{prefix}.eq') + self.ne = functools.partial(self.generator.add_statement, f'{prefix}.ne') + self.lt_u = functools.partial(self.generator.add_statement, f'{prefix}.lt_u') # 2.4.4. Memory Instructions - self.load = functools.partial(self.generator.add, 'i32.load') - self.load8_u = functools.partial(self.generator.add, 'i32.load8_u') - self.store = functools.partial(self.generator.add, 'i32.store') + self.load = functools.partial(self.generator.add_statement, f'{prefix}.load') + self.load8_u = functools.partial(self.generator.add_statement, f'{prefix}.load8_u') + self.store = functools.partial(self.generator.add_statement, f'{prefix}.store') def const(self, value: int, comment: Optional[str] = None) -> None: - self.generator.add('i32.const', f'0x{value:08x}', comment=comment) + self.generator.add_statement(f'{self.prefix}.const', f'0x{value:08x}', comment=comment) + +class Generator_i32(Generator_i32i64): + def __init__(self, generator: 'Generator') -> None: + super().__init__('i32', generator) + +class Generator_i64(Generator_i32i64): + def __init__(self, generator: 'Generator') -> None: + super().__init__('i64', generator) + +class Generator_f32f64: + def __init__(self, prefix: str, generator: 'Generator') -> None: + self.prefix = prefix + self.generator = generator + + # 2.4.1. Numeric Instructions + # fbinop + self.add = functools.partial(self.generator.add_statement, f'{prefix}.add') + + # frelop + self.eq = functools.partial(self.generator.add_statement, f'{prefix}.eq') + self.ne = functools.partial(self.generator.add_statement, f'{prefix}.ne') + + # 2.4.4. Memory Instructions + self.load = functools.partial(self.generator.add_statement, f'{prefix}.load') + self.store = functools.partial(self.generator.add_statement, f'{prefix}.store') + + def const(self, value: float, comment: Optional[str] = None) -> None: + # FIXME: Is this sufficient to guarantee the float comes across properly? + self.generator.add_statement(f'{self.prefix}.const', f'{value}', comment=comment) + +class Generator_f32(Generator_f32f64): + def __init__(self, generator: 'Generator') -> None: + super().__init__('f32', generator) + +class Generator_f64(Generator_f32f64): + def __init__(self, generator: 'Generator') -> None: + super().__init__('f64', generator) class Generator_Local: def __init__(self, generator: 'Generator') -> None: @@ -46,17 +87,17 @@ class Generator_Local: # 2.4.3. Variable Instructions def get(self, variable: VarType_Base, comment: Optional[str] = None) -> None: - self.generator.add('local.get', variable.name_ref, comment=comment) + self.generator.add_statement('local.get', variable.name_ref, comment=comment) def set(self, variable: VarType_Base, comment: Optional[str] = None) -> None: self.generator.locals.setdefault(variable.name, variable) - self.generator.add('local.set', variable.name_ref, comment=comment) + self.generator.add_statement('local.set', variable.name_ref, comment=comment) def tee(self, variable: VarType_Base, comment: Optional[str] = None) -> None: self.generator.locals.setdefault(variable.name, variable) - self.generator.add('local.tee', variable.name_ref, comment=comment) + self.generator.add_statement('local.tee', variable.name_ref, comment=comment) class GeneratorBlock: def __init__(self, generator: 'Generator', name: str) -> None: @@ -64,11 +105,11 @@ class GeneratorBlock: self.name = name def __enter__(self) -> None: - self.generator.add(self.name) + self.generator.add_statement(self.name) def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: if not exc_type: - self.generator.add('end') + self.generator.add_statement('end') class Generator: def __init__(self) -> None: @@ -76,29 +117,46 @@ class Generator: self.locals: Dict[str, VarType_Base] = {} self.i32 = Generator_i32(self) + self.i64 = Generator_i64(self) + self.f32 = Generator_f32(self) + self.f64 = Generator_f64(self) # 2.4.3 Variable Instructions self.local = Generator_Local(self) # 2.4.5 Control Instructions - self.nop = functools.partial(self.add, 'nop') - self.unreachable = functools.partial(self.add, 'unreachable') + self.nop = functools.partial(self.add_statement, 'nop') + self.unreachable = functools.partial(self.add_statement, 'unreachable') # block # loop self.if_ = functools.partial(GeneratorBlock, self, 'if') # br # br_if # br_table - self.return_ = functools.partial(self.add, 'return') + self.return_ = functools.partial(self.add_statement, 'return') # call # call_indirect def call(self, function: wasm.Function) -> None: - self.add('call', f'${function.name}') + self.add_statement('call', f'${function.name}') - def add(self, name: str, *args: str, comment: Optional[str] = None) -> None: + def add_statement(self, name: str, *args: str, comment: Optional[str] = None) -> None: self.statements.append(wasm.Statement(name, *args, comment=comment)) + def temp_var_i32(self, infix: str) -> VarType_i32: + idx = 0 + while (varname := f'__{infix}_tmp_var_{idx}__') in self.locals: + idx += 1 + + return VarType_i32(varname) + + def temp_var_u8(self, infix: str) -> VarType_u8: + idx = 0 + while (varname := f'__{infix}_tmp_var_{idx}__') in self.locals: + idx += 1 + + return VarType_u8(varname) + def func_wrapper(exported: bool = True) -> Callable[[Any], wasm.Function]: """ This wrapper will execute the function and return diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index 2ea2f8d..8e30fa0 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -411,7 +411,9 @@ def testEntry(f: bytes) -> bytes: result = Suite(code_py).run_code(b'This is a test') - assert 4 == result.returned_value + # THIS DEPENDS ON THE ALLOCATOR + # A different allocator will return a different value + assert 20 == result.returned_value @pytest.mark.integration_test def test_bytes_length(): From b6fb0d45b661b765af2a14a3c2c2df57a363cec6 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Thu, 11 Aug 2022 19:56:16 +0200 Subject: [PATCH 64/73] Implements foldl --- TODO.md | 1 - examples/fold.html | 65 ++++++++++++++++++++++ examples/fold.py | 6 ++ examples/index.html | 1 + phasm/codestyle.py | 4 ++ phasm/compiler.py | 89 ++++++++++++++++++++++++------ phasm/parser.py | 2 + phasm/wasmgenerator.py | 10 +++- tests/integration/test_builtins.py | 65 ++++++++++++++++++++++ 9 files changed, 223 insertions(+), 20 deletions(-) create mode 100644 examples/fold.html create mode 100644 examples/fold.py create mode 100644 tests/integration/test_builtins.py diff --git a/TODO.md b/TODO.md index b2ab013..29f8e80 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,4 @@ # TODO -- Implement foldl for bytes - Implement a trace() builtin for debugging - Implement a proper type matching / checking system diff --git a/examples/fold.html b/examples/fold.html new file mode 100644 index 0000000..0007e6f --- /dev/null +++ b/examples/fold.html @@ -0,0 +1,65 @@ + + + +Examples - Fold + + +

Fold

+ +List - Source - WebAssembly + +
+ + + + + + diff --git a/examples/fold.py b/examples/fold.py new file mode 100644 index 0000000..58b4b83 --- /dev/null +++ b/examples/fold.py @@ -0,0 +1,6 @@ +def u8_or(l: u8, r: u8) -> u8: + return l | r + +@exported +def foldl_u8_or_1(b: bytes) -> u8: + return foldl(u8_or, 1, b) diff --git a/examples/index.html b/examples/index.html index 46e1fc1..f01ade3 100644 --- a/examples/index.html +++ b/examples/index.html @@ -11,6 +11,7 @@

Technical

diff --git a/phasm/codestyle.py b/phasm/codestyle.py index 93853dd..f700d9b 100644 --- a/phasm/codestyle.py +++ b/phasm/codestyle.py @@ -125,6 +125,10 @@ def expression(inp: ourlang.Expression) -> str: if isinstance(inp, ourlang.AccessTupleMember): return f'{expression(inp.varref)}[{inp.member.idx}]' + if isinstance(inp, ourlang.Fold): + fold_name = 'foldl' if ourlang.Fold.Direction.LEFT == inp.dir else 'foldr' + return f'{fold_name}({inp.func.name}, {expression(inp.base)}, {expression(inp.iter)})' + raise NotImplementedError(expression, inp) def statement(inp: ourlang.Statement) -> Statements: diff --git a/phasm/compiler.py b/phasm/compiler.py index e693601..cb640bf 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -77,9 +77,10 @@ OPERATOR_MAP = { U8_OPERATOR_MAP = { # Under the hood, this is an i32 - # Implementing XOR is fine since the 3 remaining + # Implementing XOR, OR is fine since the 3 remaining # bytes stay zero after this operation '^': 'xor', + '|': 'or', } U32_OPERATOR_MAP = { @@ -254,25 +255,81 @@ def expression(wgn: WasmGenerator, inp: ourlang.Expression) -> None: return if isinstance(inp, ourlang.Fold): - mtyp = LOAD_STORE_TYPE_MAP.get(inp.base.type.__class__) - if mtyp is None: - # In the future might extend this by having structs or tuples - # as members of struct or tuples - raise NotImplementedError(expression, inp, inp.base) - - if inp.iter.type.__class__.__name__ != 'TypeBytes': - raise NotImplementedError(expression, inp, inp.iter.type) - - tmp_var = wgn.temp_var_u8(f'fold_{codestyle.type_(inp.type)}') - expression(wgn, inp.base) - wgn.local.set(tmp_var) - - # FIXME: Do a loop - wgn.unreachable(comment='Not implemented yet') + expression_fold(wgn, inp) return raise NotImplementedError(expression, inp) +def expression_fold(wgn: WasmGenerator, inp: ourlang.Fold) -> None: + """ + Compile: Fold expression + """ + mtyp = LOAD_STORE_TYPE_MAP.get(inp.base.type.__class__) + if mtyp is None: + # In the future might extend this by having structs or tuples + # as members of struct or tuples + raise NotImplementedError(expression, inp, inp.base) + + if inp.iter.type.__class__.__name__ != 'TypeBytes': + raise NotImplementedError(expression, inp, inp.iter.type) + + wgn.add_statement('nop', comment='acu :: u8') + acu_var = wgn.temp_var_u8(f'fold_{codestyle.type_(inp.type)}_acu') + wgn.add_statement('nop', comment='adr :: bytes*') + adr_var = wgn.temp_var_i32(f'fold_i32_adr') + wgn.add_statement('nop', comment='len :: i32') + len_var = wgn.temp_var_i32(f'fold_i32_len') + + wgn.add_statement('nop', comment='acu = base') + expression(wgn, inp.base) + wgn.local.set(acu_var) + + wgn.add_statement('nop', comment='adr = adr(iter)') + expression(wgn, inp.iter) + wgn.local.set(adr_var) + + wgn.add_statement('nop', comment='len = len(iter)') + wgn.local.get(adr_var) + wgn.i32.load() + wgn.local.set(len_var) + + wgn.add_statement('nop', comment='i = 0') + idx_var = wgn.temp_var_i32(f'fold_{codestyle.type_(inp.type)}_idx') + wgn.i32.const(0) + wgn.local.set(idx_var) + + wgn.add_statement('nop', comment='if i < len') + wgn.local.get(idx_var) + wgn.local.get(len_var) + wgn.i32.lt_u() + with wgn.if_(): + wgn.add_statement('nop', comment='while True') + with wgn.loop(): + wgn.add_statement('nop', comment='acu = func(acu, iter[i])') + wgn.local.get(acu_var) + wgn.local.get(adr_var) + wgn.local.get(idx_var) + wgn.call(stdlib_types.__subscript_bytes__) + wgn.add_statement('call', f'${inp.func.name}') + wgn.local.set(acu_var) + + wgn.add_statement('nop', comment='i = i + 1') + wgn.local.get(idx_var) + wgn.i32.const(1) + wgn.i32.add() + wgn.local.set(idx_var) + + wgn.add_statement('nop', comment='if i >= len: break') + wgn.local.get(idx_var) + wgn.local.get(len_var) + wgn.i32.lt_u() + wgn.br_if(0) + + # return acu + wgn.local.get(acu_var) + + return + def statement_return(wgn: WasmGenerator, inp: ourlang.StatementReturn) -> None: """ Compile: Return statement diff --git a/phasm/parser.py b/phasm/parser.py index d9fbbc1..2cb1b4a 100644 --- a/phasm/parser.py +++ b/phasm/parser.py @@ -244,6 +244,8 @@ class OurVisitor: operator = '-' elif isinstance(node.op, ast.Mult): operator = '*' + elif isinstance(node.op, ast.BitOr): + operator = '|' elif isinstance(node.op, ast.BitXor): operator = '^' else: diff --git a/phasm/wasmgenerator.py b/phasm/wasmgenerator.py index dc16589..57b1fe5 100644 --- a/phasm/wasmgenerator.py +++ b/phasm/wasmgenerator.py @@ -35,6 +35,7 @@ class Generator_i32i64: self.eq = functools.partial(self.generator.add_statement, f'{prefix}.eq') self.ne = functools.partial(self.generator.add_statement, f'{prefix}.ne') self.lt_u = functools.partial(self.generator.add_statement, f'{prefix}.lt_u') + self.ge_u = functools.partial(self.generator.add_statement, f'{prefix}.ge_u') # 2.4.4. Memory Instructions self.load = functools.partial(self.generator.add_statement, f'{prefix}.load') @@ -128,15 +129,18 @@ class Generator: self.nop = functools.partial(self.add_statement, 'nop') self.unreachable = functools.partial(self.add_statement, 'unreachable') # block - # loop + self.loop = functools.partial(GeneratorBlock, self, 'loop') self.if_ = functools.partial(GeneratorBlock, self, 'if') # br - # br_if + # br_if - see below # br_table self.return_ = functools.partial(self.add_statement, 'return') - # call + # call - see below # call_indirect + def br_if(self, idx: int) -> None: + self.add_statement('br_if', f'{idx}') + def call(self, function: wasm.Function) -> None: self.add_statement('call', f'${function.name}') diff --git a/tests/integration/test_builtins.py b/tests/integration/test_builtins.py new file mode 100644 index 0000000..4d0035c --- /dev/null +++ b/tests/integration/test_builtins.py @@ -0,0 +1,65 @@ +import sys + +import pytest + +from .helpers import Suite, write_header +from .runners import RunnerPywasm + +def setup_interpreter(phash_code: str) -> RunnerPywasm: + runner = RunnerPywasm(phash_code) + + runner.parse() + runner.compile_ast() + runner.compile_wat() + runner.compile_wasm() + runner.interpreter_setup() + runner.interpreter_load() + + write_header(sys.stderr, 'Phasm') + runner.dump_phasm_code(sys.stderr) + write_header(sys.stderr, 'Assembly') + runner.dump_wasm_wat(sys.stderr) + + return runner + +@pytest.mark.integration_test +def test_foldl_1(): + code_py = """ +def u8_or(l: u8, r: u8) -> u8: + return l | r + +@exported +def testEntry(b: bytes) -> u8: + return foldl(u8_or, 128, b) +""" + suite = Suite(code_py) + + result = suite.run_code(b'') + assert 128 == result.returned_value + + result = suite.run_code(b'\x80', runtime='pywasm') + assert 128 == result.returned_value + + result = suite.run_code(b'\x80\x40', runtime='pywasm') + assert 192 == result.returned_value + + result = suite.run_code(b'\x80\x40\x20\x10', runtime='pywasm') + assert 240 == result.returned_value + + result = suite.run_code(b'\x80\x40\x20\x10\x08\x04\x02\x01', runtime='pywasm') + assert 255 == result.returned_value + +@pytest.mark.integration_test +def test_foldl_2(): + code_py = """ +def xor(l: u8, r: u8) -> u8: + return l ^ r + +@exported +def testEntry(a: bytes, b: bytes) -> u8: + return foldl(xor, 0, a) ^ foldl(xor, 0, b) +""" + suite = Suite(code_py) + + result = suite.run_code(b'\x55\x0F', b'\x33\x80') + assert 233 == result.returned_value From ad6ca71c53fb91e5c563be2677fc3c23b9a1bf06 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Fri, 12 Aug 2022 21:50:42 +0200 Subject: [PATCH 65/73] More bitwise ops. Steps towards CRC32 --- examples/crc32.py | 9 ++++--- phasm/compiler.py | 7 +++++- phasm/parser.py | 2 ++ tests/integration/test_examples.py | 39 ++++++++++++++++++++++++++++++ tests/integration/test_simple.py | 30 ++++++++++++++++++++++- 5 files changed, 81 insertions(+), 6 deletions(-) create mode 100644 tests/integration/test_examples.py diff --git a/examples/crc32.py b/examples/crc32.py index 87b6912..ff4d2f9 100644 --- a/examples/crc32.py +++ b/examples/crc32.py @@ -13,7 +13,8 @@ # return crc32; # } -_CRC32_Table: [u32] = [ +# Might be the wrong table +_CRC32_Table: u32[256] = [ 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, @@ -80,9 +81,9 @@ _CRC32_Table: [u32] = [ 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351, ] -def _crc32_f(val: u32, byt: u8) -> u32: - return (val >> 8) ^ _CRC32_Table[(val ^ byt) & 0xFF] +def _crc32_f(crc: u32, byt: u8) -> u32: + return (crc >> 8) ^ _CRC32_Table[(crc & 0xFF) ^ byt] @exported def crc32(data: bytes) -> u32: - return 0xFFFFFFFF ^ foldli(_crc32_f, 0xFFFFFFFF, data) + return 0xFFFFFFFF ^ foldl(_crc32_f, 0xFFFFFFFF, data) diff --git a/phasm/compiler.py b/phasm/compiler.py index cb640bf..46a9dfe 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -77,10 +77,11 @@ OPERATOR_MAP = { U8_OPERATOR_MAP = { # Under the hood, this is an i32 - # Implementing XOR, OR is fine since the 3 remaining + # Implementing XOR, OR, AND is fine since the 3 remaining # bytes stay zero after this operation '^': 'xor', '|': 'or', + '&': 'and', } U32_OPERATOR_MAP = { @@ -89,6 +90,8 @@ U32_OPERATOR_MAP = { '<=': 'le_u', '>=': 'ge_u', '^': 'xor', + '|': 'or', + '&': 'and', } U64_OPERATOR_MAP = { @@ -97,6 +100,8 @@ U64_OPERATOR_MAP = { '<=': 'le_u', '>=': 'ge_u', '^': 'xor', + '|': 'or', + '&': 'and', } I32_OPERATOR_MAP = { diff --git a/phasm/parser.py b/phasm/parser.py index 2cb1b4a..fdf63c0 100644 --- a/phasm/parser.py +++ b/phasm/parser.py @@ -248,6 +248,8 @@ class OurVisitor: operator = '|' elif isinstance(node.op, ast.BitXor): operator = '^' + elif isinstance(node.op, ast.BitAnd): + operator = '&' else: raise NotImplementedError(f'Operator {node.op}') diff --git a/tests/integration/test_examples.py b/tests/integration/test_examples.py new file mode 100644 index 0000000..b3b278d --- /dev/null +++ b/tests/integration/test_examples.py @@ -0,0 +1,39 @@ +import binascii +import struct + +import pytest + +from .helpers import Suite + +@pytest.mark.integration_test +def test_crc32(): + # FIXME: Stub + # crc = 0xFFFFFFFF + # byt = 0x61 + # => (crc >> 8) ^ _CRC32_Table[(crc & 0xFF) ^ byt] + # (crc >> 8) = 0x00FFFFFF + # => 0x00FFFFFF ^ _CRC32_Table[(crc & 0xFF) ^ byt] + # (crc & 0xFF) = 0xFF + # => 0x00FFFFFF ^ _CRC32_Table[0xFF ^ byt] + # 0xFF ^ 0x61 = 0x9E + # => 0x00FFFFFF ^ _CRC32_Table[0x9E] + # _CRC32_Table[0x9E] = 0x17b7be43 + # => 0x00FFFFFF ^ 0x17b7be43 + + code_py = """ +def _crc32_f(crc: u32, byt: u8) -> u32: + return 16777215 ^ 397917763 + +def testEntry(data: bytes) -> u32: + return 4294967295 ^ _crc32_f(4294967295, data[0]) +""" + exp_result = binascii.crc32(b'a') + + result = Suite(code_py).run_code(b'a') + + # exp_result returns a unsigned integer, as is proper + exp_data = struct.pack('I', exp_result) + # ints extracted from WebAssembly are always signed + data = struct.pack('i', result.returned_value) + + assert exp_data == data diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index 8e30fa0..2669eee 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -62,7 +62,21 @@ def testEntry() -> {type_}: @pytest.mark.integration_test @pytest.mark.parametrize('type_', ['u8', 'u32', 'u64']) -def test_xor(type_): +def test_bitwise_or(type_): + code_py = f""" +@exported +def testEntry() -> {type_}: + return 10 | 3 +""" + + result = Suite(code_py).run_code() + + assert 11 == result.returned_value + assert TYPE_MAP[type_] == type(result.returned_value) + +@pytest.mark.integration_test +@pytest.mark.parametrize('type_', ['u8', 'u32', 'u64']) +def test_bitwise_xor(type_): code_py = f""" @exported def testEntry() -> {type_}: @@ -74,6 +88,20 @@ def testEntry() -> {type_}: assert 9 == result.returned_value assert TYPE_MAP[type_] == type(result.returned_value) +@pytest.mark.integration_test +@pytest.mark.parametrize('type_', ['u8', 'u32', 'u64']) +def test_bitwise_and(type_): + code_py = f""" +@exported +def testEntry() -> {type_}: + return 10 & 3 +""" + + result = Suite(code_py).run_code() + + assert 2 == result.returned_value + assert TYPE_MAP[type_] == type(result.returned_value) + @pytest.mark.integration_test @pytest.mark.parametrize('type_', ['f32', 'f64']) def test_buildins_sqrt(type_): From 75d7e0551918a3199563fe2d94715c77dd18fd8f Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Tue, 16 Aug 2022 20:38:41 +0200 Subject: [PATCH 66/73] First uint cast, more options for folding --- phasm/codestyle.py | 3 +++ phasm/compiler.py | 5 +++++ phasm/parser.py | 32 +++++++++++++++++++++++++----- tests/integration/test_builtins.py | 15 ++++++++++++++ 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/phasm/codestyle.py b/phasm/codestyle.py index f700d9b..e2e64f0 100644 --- a/phasm/codestyle.py +++ b/phasm/codestyle.py @@ -97,6 +97,9 @@ def expression(inp: ourlang.Expression) -> str: or inp.operator in ourlang.WEBASSEMBLY_BUILDIN_BYTES_OPS): return f'{inp.operator}({expression(inp.right)})' + if inp.operator == 'cast': + return f'{type_(inp.type)}({expression(inp.right)})' + return f'{inp.operator}{expression(inp.right)}' if isinstance(inp, ourlang.BinaryOp): diff --git a/phasm/compiler.py b/phasm/compiler.py index 46a9dfe..4f810b6 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -219,6 +219,11 @@ def expression(wgn: WasmGenerator, inp: ourlang.Expression) -> None: wgn.i32.load() return + if inp.operator == 'cast': + if isinstance(inp.type, typing.TypeUInt32) and isinstance(inp.right.type, typing.TypeUInt8): + # Nothing to do, you can use an u8 value as a u32 no problem + return + raise NotImplementedError(expression, inp.type, inp.operator) if isinstance(inp, ourlang.FunctionCall): diff --git a/phasm/parser.py b/phasm/parser.py index fdf63c0..14e1ba2 100644 --- a/phasm/parser.py +++ b/phasm/parser.py @@ -380,6 +380,20 @@ class OurVisitor: 'sqrt', self.visit_Module_FunctionDef_expr(module, function, our_locals, exp_type, node.args[0]), ) + elif node.func.id == 'u32': + if not isinstance(exp_type, TypeUInt32): + _raise_static_error(node, f'Cannot make {node.func.id} result in {exp_type}') + + if 1 != len(node.args): + _raise_static_error(node, f'Function {node.func.id} requires 1 arguments but {len(node.args)} are given') + + # FIXME: This is a stub, proper casting is todo + + return UnaryOp( + exp_type, + 'cast', + self.visit_Module_FunctionDef_expr(module, function, our_locals, module.types['u8'], node.args[0]), + ) elif node.func.id == 'len': if not isinstance(exp_type, TypeInt32): _raise_static_error(node, f'Cannot make {node.func.id} result in {exp_type}') @@ -398,14 +412,9 @@ class OurVisitor: # In the future, we should probably infer the type of the second argument, # and use it as expected types for the other u8s and the Iterable[u8] (i.e. bytes) - if not isinstance(exp_type, TypeUInt8): - _raise_static_error(node, f'Cannot make {node.func.id} result in {exp_type} - not implemented yet') - if 3 != len(node.args): _raise_static_error(node, f'Function {node.func.id} requires 3 arguments but {len(node.args)} are given') - t_u8 = module.types['u8'] - # TODO: This is not generic subnode = node.args[0] if not isinstance(subnode, ast.Name): @@ -415,6 +424,19 @@ class OurVisitor: if subnode.id not in module.functions: _raise_static_error(subnode, 'Reference to undefined function') func = module.functions[subnode.id] + if 2 != len(func.posonlyargs): + _raise_static_error(node, f'Function {node.func.id} requires a function with 2 arguments but a function with {len(func.posonlyargs)} args is given') + + if exp_type.__class__ != func.returns.__class__: + _raise_static_error(node, f'Expected {codestyle.type_(exp_type)}, {func.name} actually returns {codestyle.type_(func.returns)}') + + if func.returns.__class__ != func.posonlyargs[0][1].__class__: + _raise_static_error(node, f'Expected a foldable function, {func.name} returns a {codestyle.type_(func.returns)} but expects a {codestyle.type_(func.posonlyargs[0][1])}') + + t_u8 = module.types['u8'] + + if t_u8.__class__ != func.posonlyargs[1][1].__class__: + _raise_static_error(node, 'Only folding over bytes (u8) is supported at this time') return Fold( exp_type, diff --git a/tests/integration/test_builtins.py b/tests/integration/test_builtins.py index 4d0035c..4b84197 100644 --- a/tests/integration/test_builtins.py +++ b/tests/integration/test_builtins.py @@ -63,3 +63,18 @@ def testEntry(a: bytes, b: bytes) -> u8: result = suite.run_code(b'\x55\x0F', b'\x33\x80') assert 233 == result.returned_value + +@pytest.mark.integration_test +def test_foldl_3(): + code_py = """ +def xor(l: u32, r: u8) -> u32: + return l ^ u32(r) + +@exported +def testEntry(a: bytes) -> u32: + return foldl(xor, 0, a) +""" + suite = Suite(code_py) + + result = suite.run_code(b'\x55\x0F\x33\x80') + assert 233 == result.returned_value From d0511602441ea7706bd1e6aeafbdfbf6a8bc1aa5 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Tue, 16 Aug 2022 20:55:20 +0200 Subject: [PATCH 67/73] Bit shifting --- examples/crc32.py | 2 +- phasm/compiler.py | 9 ++++++++- phasm/parser.py | 12 +++++++----- tests/integration/test_simple.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/examples/crc32.py b/examples/crc32.py index ff4d2f9..781bf21 100644 --- a/examples/crc32.py +++ b/examples/crc32.py @@ -82,7 +82,7 @@ _CRC32_Table: u32[256] = [ ] def _crc32_f(crc: u32, byt: u8) -> u32: - return (crc >> 8) ^ _CRC32_Table[(crc & 0xFF) ^ byt] + return (crc >> 8) ^ _CRC32_Table[(crc & 0xFF) ^ u32(byt)] @exported def crc32(data: bytes) -> u32: diff --git a/phasm/compiler.py b/phasm/compiler.py index 4f810b6..723da8a 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -77,8 +77,11 @@ OPERATOR_MAP = { U8_OPERATOR_MAP = { # Under the hood, this is an i32 - # Implementing XOR, OR, AND is fine since the 3 remaining + # Implementing Right Shift XOR, OR, AND is fine since the 3 remaining # bytes stay zero after this operation + # Since it's unsigned an unsigned value, Logical or Arithmetic shift right + # are the same operation + '>>': 'shr_u', '^': 'xor', '|': 'or', '&': 'and', @@ -89,6 +92,8 @@ U32_OPERATOR_MAP = { '>': 'gt_u', '<=': 'le_u', '>=': 'ge_u', + '<<': 'shl', + '>>': 'shr_u', '^': 'xor', '|': 'or', '&': 'and', @@ -99,6 +104,8 @@ U64_OPERATOR_MAP = { '>': 'gt_u', '<=': 'le_u', '>=': 'ge_u', + '<<': 'shl', + '>>': 'shr_u', '^': 'xor', '|': 'or', '&': 'and', diff --git a/phasm/parser.py b/phasm/parser.py index 14e1ba2..f1de44f 100644 --- a/phasm/parser.py +++ b/phasm/parser.py @@ -244,6 +244,10 @@ class OurVisitor: operator = '-' elif isinstance(node.op, ast.Mult): operator = '*' + elif isinstance(node.op, ast.LShift): + operator = '<<' + elif isinstance(node.op, ast.RShift): + operator = '>>' elif isinstance(node.op, ast.BitOr): operator = '|' elif isinstance(node.op, ast.BitXor): @@ -433,16 +437,14 @@ class OurVisitor: if func.returns.__class__ != func.posonlyargs[0][1].__class__: _raise_static_error(node, f'Expected a foldable function, {func.name} returns a {codestyle.type_(func.returns)} but expects a {codestyle.type_(func.posonlyargs[0][1])}') - t_u8 = module.types['u8'] - - if t_u8.__class__ != func.posonlyargs[1][1].__class__: + if module.types['u8'].__class__ != func.posonlyargs[1][1].__class__: _raise_static_error(node, 'Only folding over bytes (u8) is supported at this time') return Fold( exp_type, Fold.Direction.LEFT, func, - self.visit_Module_FunctionDef_expr(module, function, our_locals, t_u8, node.args[1]), + self.visit_Module_FunctionDef_expr(module, function, our_locals, func.returns, node.args[1]), self.visit_Module_FunctionDef_expr(module, function, our_locals, module.types['bytes'], node.args[2]), ) else: @@ -550,7 +552,7 @@ class OurVisitor: _raise_static_error(node, 'Expected integer value') if node.value < 0 or node.value > 255: - _raise_static_error(node, 'Integer value out of range') + _raise_static_error(node, f'Integer value out of range; expected 0..255, actual {node.value}') return ConstantUInt8(exp_type, node.value) diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index 2669eee..b6bfbc5 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -60,6 +60,34 @@ def testEntry() -> {type_}: assert 7 == result.returned_value assert TYPE_MAP[type_] == type(result.returned_value) +@pytest.mark.integration_test +@pytest.mark.parametrize('type_', ['u32', 'u64']) # FIXME: Support u8, requires an extra AND operation +def test_logical_left_shift(type_): + code_py = f""" +@exported +def testEntry() -> {type_}: + return 10 << 3 +""" + + result = Suite(code_py).run_code() + + assert 80 == result.returned_value + assert TYPE_MAP[type_] == type(result.returned_value) + +@pytest.mark.integration_test +@pytest.mark.parametrize('type_', ['u8', 'u32', 'u64']) +def test_logical_right_shift(type_): + code_py = f""" +@exported +def testEntry() -> {type_}: + return 10 >> 3 +""" + + result = Suite(code_py).run_code() + + assert 1 == result.returned_value + assert TYPE_MAP[type_] == type(result.returned_value) + @pytest.mark.integration_test @pytest.mark.parametrize('type_', ['u8', 'u32', 'u64']) def test_bitwise_or(type_): From bac17f47eb6bf774f66069f2d54aa64fcebe3ed9 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Tue, 16 Aug 2022 21:25:03 +0200 Subject: [PATCH 68/73] ModuleConstant definitions and references --- phasm/codestyle.py | 14 ++++++ phasm/compiler.py | 16 +++++-- phasm/ourlang.py | 37 +++++++++++++-- phasm/parser.py | 74 ++++++++++++++++++++++------- tests/integration/test_constants.py | 17 +++++++ 5 files changed, 135 insertions(+), 23 deletions(-) create mode 100644 tests/integration/test_constants.py diff --git a/phasm/codestyle.py b/phasm/codestyle.py index e2e64f0..4d19bd6 100644 --- a/phasm/codestyle.py +++ b/phasm/codestyle.py @@ -73,6 +73,12 @@ def struct_definition(inp: typing.TypeStruct) -> str: return result +def constant_definition(inp: ourlang.ModuleConstantDef) -> str: + """ + Render: Module Constant's definition + """ + return f'{inp.name}: {type_(inp.type)} = {expression(inp.constant)}\n' + def expression(inp: ourlang.Expression) -> str: """ Render: A Phasm expression @@ -132,6 +138,9 @@ def expression(inp: ourlang.Expression) -> str: fold_name = 'foldl' if ourlang.Fold.Direction.LEFT == inp.dir else 'foldr' return f'{fold_name}({inp.func.name}, {expression(inp.base)}, {expression(inp.iter)})' + if isinstance(inp, ourlang.ModuleConstantReference): + return inp.definition.name + raise NotImplementedError(expression, inp) def statement(inp: ourlang.Statement) -> Statements: @@ -199,6 +208,11 @@ def module(inp: ourlang.Module) -> str: result += '\n' result += struct_definition(struct) + for cdef in inp.constant_defs.values(): + if result: + result += '\n' + result += constant_definition(cdef) + for func in inp.functions.values(): if func.lineno < 0: # Buildin (-2) or auto generated (-1) diff --git a/phasm/compiler.py b/phasm/compiler.py index 723da8a..9490ad4 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -275,6 +275,16 @@ def expression(wgn: WasmGenerator, inp: ourlang.Expression) -> None: expression_fold(wgn, inp) return + if isinstance(inp, ourlang.ModuleConstantReference): + mtyp = LOAD_STORE_TYPE_MAP.get(inp.type.__class__) + if mtyp is None: + # In the future might extend this by having structs or tuples + # as members of struct or tuples + raise NotImplementedError(expression, inp, inp.type) + + expression(wgn, inp.definition.constant) + return + raise NotImplementedError(expression, inp) def expression_fold(wgn: WasmGenerator, inp: ourlang.Fold) -> None: @@ -293,9 +303,9 @@ def expression_fold(wgn: WasmGenerator, inp: ourlang.Fold) -> None: wgn.add_statement('nop', comment='acu :: u8') acu_var = wgn.temp_var_u8(f'fold_{codestyle.type_(inp.type)}_acu') wgn.add_statement('nop', comment='adr :: bytes*') - adr_var = wgn.temp_var_i32(f'fold_i32_adr') + adr_var = wgn.temp_var_i32('fold_i32_adr') wgn.add_statement('nop', comment='len :: i32') - len_var = wgn.temp_var_i32(f'fold_i32_len') + len_var = wgn.temp_var_i32('fold_i32_len') wgn.add_statement('nop', comment='acu = base') expression(wgn, inp.base) @@ -345,8 +355,6 @@ def expression_fold(wgn: WasmGenerator, inp: ourlang.Fold) -> None: # return acu wgn.local.get(acu_var) - return - def statement_return(wgn: WasmGenerator, inp: ourlang.StatementReturn) -> None: """ Compile: Return statement diff --git a/phasm/ourlang.py b/phasm/ourlang.py index 4d01b16..4e245a2 100644 --- a/phasm/ourlang.py +++ b/phasm/ourlang.py @@ -258,6 +258,18 @@ class Fold(Expression): self.base = base self.iter = iter_ +class ModuleConstantReference(Expression): + """ + An reference to a module constant expression within a statement + """ + __slots__ = ('definition', ) + + definition: 'ModuleConstantDef' + + def __init__(self, type_: TypeBase, definition: 'ModuleConstantDef') -> None: + super().__init__(type_) + self.definition = definition + class Statement: """ A statement within a function @@ -360,15 +372,33 @@ class TupleConstructor(Function): self.tuple = tuple_ +class ModuleConstantDef: + """ + A constant definition within a module + """ + __slots__ = ('name', 'lineno', 'type', 'constant', ) + + name: str + lineno: int + type: TypeBase + constant: Constant + + def __init__(self, name: str, lineno: int, type_: TypeBase, constant: Constant) -> None: + self.name = name + self.lineno = lineno + self.type = type_ + self.constant = constant + class Module: """ A module is a file and consists of functions """ - __slots__ = ('types', 'functions', 'structs', ) + __slots__ = ('types', 'structs', 'constant_defs', 'functions',) types: Dict[str, TypeBase] - functions: Dict[str, Function] structs: Dict[str, TypeStruct] + constant_defs: Dict[str, ModuleConstantDef] + functions: Dict[str, Function] def __init__(self) -> None: self.types = { @@ -382,5 +412,6 @@ class Module: 'f64': TypeFloat64(), 'bytes': TypeBytes(), } - self.functions = {} self.structs = {} + self.constant_defs = {} + self.functions = {} diff --git a/phasm/parser.py b/phasm/parser.py index f1de44f..30d2b26 100644 --- a/phasm/parser.py +++ b/phasm/parser.py @@ -32,16 +32,19 @@ from .ourlang import ( Expression, AccessBytesIndex, AccessStructMember, AccessTupleMember, BinaryOp, + Constant, ConstantFloat32, ConstantFloat64, ConstantInt32, ConstantInt64, ConstantUInt8, ConstantUInt32, ConstantUInt64, FunctionCall, StructConstructor, TupleConstructor, UnaryOp, VariableReference, - Fold, + Fold, ModuleConstantReference, Statement, StatementIf, StatementPass, StatementReturn, + + ModuleConstantDef, ) def phasm_parse(source: str) -> Module: @@ -80,13 +83,13 @@ class OurVisitor: for stmt in node.body: res = self.pre_visit_Module_stmt(module, stmt) - if isinstance(res, Function): - if res.name in module.functions: + if isinstance(res, ModuleConstantDef): + if res.name in module.constant_defs: raise StaticError( - f'{res.name} already defined on line {module.functions[res.name].lineno}' + f'{res.name} already defined on line {module.constant_defs[res.name].lineno}' ) - module.functions[res.name] = res + module.constant_defs[res.name] = res if isinstance(res, TypeStruct): if res.name in module.structs: @@ -98,6 +101,14 @@ class OurVisitor: constructor = StructConstructor(res) module.functions[constructor.name] = constructor + if isinstance(res, Function): + if res.name in module.functions: + raise StaticError( + f'{res.name} already defined on line {module.functions[res.name].lineno}' + ) + + module.functions[res.name] = res + # Second pass for the function bodies for stmt in node.body: @@ -105,13 +116,16 @@ class OurVisitor: return module - def pre_visit_Module_stmt(self, module: Module, node: ast.stmt) -> Union[Function, TypeStruct]: + def pre_visit_Module_stmt(self, module: Module, node: ast.stmt) -> Union[Function, TypeStruct, ModuleConstantDef]: if isinstance(node, ast.FunctionDef): return self.pre_visit_Module_FunctionDef(module, node) if isinstance(node, ast.ClassDef): return self.pre_visit_Module_ClassDef(module, node) + if isinstance(node, ast.AnnAssign): + return self.pre_visit_Module_AnnAssign(module, node) + raise NotImplementedError(f'{node} on Module') def pre_visit_Module_FunctionDef(self, module: Module, node: ast.FunctionDef) -> Function: @@ -184,6 +198,25 @@ class OurVisitor: return struct + def pre_visit_Module_AnnAssign(self, module: Module, node: ast.AnnAssign) -> ModuleConstantDef: + if not isinstance(node.target, ast.Name): + _raise_static_error(node, 'Must be name') + if not isinstance(node.target.ctx, ast.Store): + _raise_static_error(node, 'Must be load context') + if not isinstance(node.value, ast.Constant): + _raise_static_error(node, 'Must be constant') + + exp_type = self.visit_type(module, node.annotation) + + constant = ModuleConstantDef( + node.target.id, + node.lineno, + exp_type, + self.visit_Module_Constant(module, exp_type, node.value) + ) + + return constant + def visit_Module_stmt(self, module: Module, node: ast.stmt) -> None: if isinstance(node, ast.FunctionDef): self.visit_Module_FunctionDef(module, node) @@ -192,6 +225,9 @@ class OurVisitor: if isinstance(node, ast.ClassDef): return + if isinstance(node, ast.AnnAssign): + return + raise NotImplementedError(f'{node} on Module') def visit_Module_FunctionDef(self, module: Module, node: ast.FunctionDef) -> None: @@ -308,8 +344,8 @@ class OurVisitor: return self.visit_Module_FunctionDef_Call(module, function, our_locals, exp_type, node) if isinstance(node, ast.Constant): - return self.visit_Module_FunctionDef_Constant( - module, function, exp_type, node, + return self.visit_Module_Constant( + module, exp_type, node, ) if isinstance(node, ast.Attribute): @@ -326,14 +362,21 @@ class OurVisitor: if not isinstance(node.ctx, ast.Load): _raise_static_error(node, 'Must be load context') - if node.id not in our_locals: - _raise_static_error(node, 'Undefined variable') + if node.id in our_locals: + act_type = our_locals[node.id] + if exp_type != act_type: + _raise_static_error(node, f'Expected {codestyle.type_(exp_type)}, {node.id} is actually {codestyle.type_(act_type)}') - act_type = our_locals[node.id] - if exp_type != act_type: - _raise_static_error(node, f'Expected {codestyle.type_(exp_type)}, {node.id} is actually {codestyle.type_(act_type)}') + return VariableReference(act_type, node.id) - return VariableReference(act_type, node.id) + if node.id in module.constant_defs: + cdef = module.constant_defs[node.id] + if exp_type != cdef.type: + _raise_static_error(node, f'Expected {codestyle.type_(exp_type)}, {node.id} is actually {codestyle.type_(act_type)}') + + return ModuleConstantReference(exp_type, cdef) + + _raise_static_error(node, f'Undefined variable {node.id}') if isinstance(node, ast.Tuple): if not isinstance(node.ctx, ast.Load): @@ -541,9 +584,8 @@ class OurVisitor: _raise_static_error(node, f'Cannot take index of {node_typ} {node.value.id}') - def visit_Module_FunctionDef_Constant(self, module: Module, function: Function, exp_type: TypeBase, node: ast.Constant) -> Expression: + def visit_Module_Constant(self, module: Module, exp_type: TypeBase, node: ast.Constant) -> Constant: del module - del function _not_implemented(node.kind is None, 'Constant.kind') diff --git a/tests/integration/test_constants.py b/tests/integration/test_constants.py new file mode 100644 index 0000000..1593915 --- /dev/null +++ b/tests/integration/test_constants.py @@ -0,0 +1,17 @@ +import pytest + +from .helpers import Suite + +@pytest.mark.integration_test +def test_return(): + code_py = """ +CONSTANT: i32 = 13 + +@exported +def testEntry() -> i32: + return CONSTANT * 5 +""" + + result = Suite(code_py).run_code() + + assert 65 == result.returned_value From c4ee2ab3dc52c638ada69302906899a04ff2f449 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Wed, 17 Aug 2022 21:07:33 +0200 Subject: [PATCH 69/73] Memory initialization is now done during compilation Also, the user can now define tuple module constants --- TODO.md | 2 + examples/buffer.html | 3 - examples/fold.html | 3 - phasm/codestyle.py | 6 ++ phasm/compiler.py | 83 ++++++++++++++++++++++- phasm/ourlang.py | 48 +++++++++++-- phasm/parser.py | 56 ++++++++++++--- phasm/stdlib/alloc.py | 37 +--------- tests/integration/helpers.py | 2 - tests/integration/test_constants.py | 37 +++++++++- tests/integration/test_static_checking.py | 27 ++++++++ tests/integration/test_stdlib_alloc.py | 46 ------------- 12 files changed, 243 insertions(+), 107 deletions(-) diff --git a/TODO.md b/TODO.md index 29f8e80..41ae3da 100644 --- a/TODO.md +++ b/TODO.md @@ -2,3 +2,5 @@ - Implement a trace() builtin for debugging - Implement a proper type matching / checking system +- Check if we can use DataView in the Javascript examples, e.g. with setUint32 +- Storing u8 in memory still claims 32 bits (since that's what you need in local variables). However, using load8_u / loadu_s we can optimize this. diff --git a/examples/buffer.html b/examples/buffer.html index 3264f9b..cf4e442 100644 --- a/examples/buffer.html +++ b/examples/buffer.html @@ -30,9 +30,6 @@ WebAssembly.instantiateStreaming(fetch('buffer.wasm'), importObject) // Allocate room within the memory of the WebAssembly class let size = 8; - stdlib_alloc___init__ = app.instance.exports['stdlib.alloc.__init__'] - stdlib_alloc___init__() - stdlib_types___alloc_bytes__ = app.instance.exports['stdlib.types.__alloc_bytes__'] let offset = stdlib_types___alloc_bytes__(size) diff --git a/examples/fold.html b/examples/fold.html index 0007e6f..d47e580 100644 --- a/examples/fold.html +++ b/examples/fold.html @@ -31,9 +31,6 @@ function log(txt) WebAssembly.instantiateStreaming(fetch('fold.wasm'), importObject) .then(app => { - stdlib_alloc___init__ = app.instance.exports['stdlib.alloc.__init__'] - stdlib_alloc___init__() - stdlib_types___alloc_bytes__ = app.instance.exports['stdlib.types.__alloc_bytes__'] let offset0 = stdlib_types___alloc_bytes__(0); diff --git a/phasm/codestyle.py b/phasm/codestyle.py index 4d19bd6..a09b524 100644 --- a/phasm/codestyle.py +++ b/phasm/codestyle.py @@ -94,6 +94,12 @@ def expression(inp: ourlang.Expression) -> str: # could not fit in the given float type return str(inp.value) + if isinstance(inp, ourlang.ConstantTuple): + return '(' + ', '.join( + expression(x) + for x in inp.value + ) + ', )' + if isinstance(inp, ourlang.VariableReference): return str(inp.name) diff --git a/phasm/compiler.py b/phasm/compiler.py index 9490ad4..1f14aae 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -1,6 +1,8 @@ """ This module contains the code to convert parsed Ourlang into WebAssembly code """ +import struct + from . import codestyle from . import ourlang from . import typing @@ -276,6 +278,15 @@ def expression(wgn: WasmGenerator, inp: ourlang.Expression) -> None: return if isinstance(inp, ourlang.ModuleConstantReference): + if isinstance(inp.type, typing.TypeTuple): + assert isinstance(inp.definition.constant, ourlang.ConstantTuple) + assert inp.definition.data_block is not None, 'Combined values are memory stored' + assert inp.definition.data_block.address is not None, 'Value not allocated' + wgn.i32.const(inp.definition.data_block.address) + return + + assert inp.definition.data_block is None, 'Primitives are not memory stored' + mtyp = LOAD_STORE_TYPE_MAP.get(inp.type.__class__) if mtyp is None: # In the future might extend this by having structs or tuples @@ -448,12 +459,83 @@ def function(inp: ourlang.Function) -> wasm.Function: wgn.statements ) +def module_data_u8(inp: int) -> bytes: + """ + Compile: module data, u8 value + + # FIXME: All u8 values are stored as u32 + """ + return struct.pack(' bytes: + """ + Compile: module data, u32 value + """ + return struct.pack(' bytes: + """ + Compile: module data, u64 value + """ + return struct.pack(' bytes: + """ + Compile: module data + """ + unalloc_ptr = stdlib_alloc.UNALLOC_PTR + + allocated_data = b'' + + for block in inp.blocks: + block.address = unalloc_ptr + 4 # 4 bytes for allocator header + + data_list = [] + + for constant in block.data: + if isinstance(constant, ourlang.ConstantUInt8): + data_list.append(module_data_u8(constant.value)) + continue + + if isinstance(constant, ourlang.ConstantUInt32): + data_list.append(module_data_u32(constant.value)) + continue + + if isinstance(constant, ourlang.ConstantUInt64): + data_list.append(module_data_u64(constant.value)) + continue + + raise NotImplementedError(constant) + + block_data = b''.join(data_list) + + allocated_data += module_data_u32(len(block_data)) + block_data + + unalloc_ptr += 4 + len(block_data) + + return ( + # Store that we've initialized the memory + module_data_u32(stdlib_alloc.IDENTIFIER) + # Store the first reserved i32 + + module_data_u32(0) + # Store the pointer towards the first free block + # In this case, 0 since we haven't freed any blocks yet + + module_data_u32(0) + # Store the pointer towards the first unallocated block + # In this case the end of the stdlib.alloc header at the start + + module_data_u32(unalloc_ptr) + # Store the actual data + + allocated_data + ) + def module(inp: ourlang.Module) -> wasm.Module: """ Compile: module """ result = wasm.Module() + result.memory.data = module_data(inp.data) + result.imports = [ import_(x) for x in inp.functions.values() @@ -461,7 +543,6 @@ def module(inp: ourlang.Module) -> wasm.Module: ] result.functions = [ - stdlib_alloc.__init__, stdlib_alloc.__find_free_block__, stdlib_alloc.__alloc__, stdlib_types.__alloc_bytes__, diff --git a/phasm/ourlang.py b/phasm/ourlang.py index 4e245a2..d7cfffe 100644 --- a/phasm/ourlang.py +++ b/phasm/ourlang.py @@ -1,7 +1,7 @@ """ Contains the syntax tree for ourlang """ -from typing import Dict, List, Tuple +from typing import Dict, List, Tuple, Optional import enum @@ -123,6 +123,18 @@ class ConstantFloat64(Constant): super().__init__(type_) self.value = value +class ConstantTuple(Constant): + """ + A Tuple constant value expression within a statement + """ + __slots__ = ('value', ) + + value: List[Constant] + + def __init__(self, type_: TypeTuple, value: List[Constant]) -> None: + super().__init__(type_) + self.value = value + class VariableReference(Expression): """ An variable reference expression within a statement @@ -376,25 +388,52 @@ class ModuleConstantDef: """ A constant definition within a module """ - __slots__ = ('name', 'lineno', 'type', 'constant', ) + __slots__ = ('name', 'lineno', 'type', 'constant', 'data_block', ) name: str lineno: int type: TypeBase constant: Constant + data_block: Optional['ModuleDataBlock'] - def __init__(self, name: str, lineno: int, type_: TypeBase, constant: Constant) -> None: + def __init__(self, name: str, lineno: int, type_: TypeBase, constant: Constant, data_block: Optional['ModuleDataBlock']) -> None: self.name = name self.lineno = lineno self.type = type_ self.constant = constant + self.data_block = data_block + +class ModuleDataBlock: + """ + A single allocated block for module data + """ + __slots__ = ('data', 'address', ) + + data: List[Constant] + address: Optional[int] + + def __init__(self, data: List[Constant]) -> None: + self.data = data + self.address = None + +class ModuleData: + """ + The data for when a module is loaded into memory + """ + __slots__ = ('blocks', ) + + blocks: List[ModuleDataBlock] + + def __init__(self) -> None: + self.blocks = [] class Module: """ A module is a file and consists of functions """ - __slots__ = ('types', 'structs', 'constant_defs', 'functions',) + __slots__ = ('data', 'types', 'structs', 'constant_defs', 'functions',) + data: ModuleData types: Dict[str, TypeBase] structs: Dict[str, TypeStruct] constant_defs: Dict[str, ModuleConstantDef] @@ -412,6 +451,7 @@ class Module: 'f64': TypeFloat64(), 'bytes': TypeBytes(), } + self.data = ModuleData() self.structs = {} self.constant_defs = {} self.functions = {} diff --git a/phasm/parser.py b/phasm/parser.py index 30d2b26..a9f2f63 100644 --- a/phasm/parser.py +++ b/phasm/parser.py @@ -26,7 +26,7 @@ from .exceptions import StaticError from .ourlang import ( WEBASSEMBLY_BUILDIN_FLOAT_OPS, - Module, + Module, ModuleDataBlock, Function, Expression, @@ -35,6 +35,8 @@ from .ourlang import ( Constant, ConstantFloat32, ConstantFloat64, ConstantInt32, ConstantInt64, ConstantUInt8, ConstantUInt32, ConstantUInt64, + ConstantTuple, + FunctionCall, StructConstructor, TupleConstructor, UnaryOp, VariableReference, @@ -203,19 +205,51 @@ class OurVisitor: _raise_static_error(node, 'Must be name') if not isinstance(node.target.ctx, ast.Store): _raise_static_error(node, 'Must be load context') - if not isinstance(node.value, ast.Constant): - _raise_static_error(node, 'Must be constant') exp_type = self.visit_type(module, node.annotation) - constant = ModuleConstantDef( - node.target.id, - node.lineno, - exp_type, - self.visit_Module_Constant(module, exp_type, node.value) - ) + if isinstance(exp_type, TypeInt32): + if not isinstance(node.value, ast.Constant): + _raise_static_error(node, 'Must be constant') - return constant + constant = ModuleConstantDef( + node.target.id, + node.lineno, + exp_type, + self.visit_Module_Constant(module, exp_type, node.value), + None, + ) + return constant + + if isinstance(exp_type, TypeTuple): + if not isinstance(node.value, ast.Tuple): + _raise_static_error(node, 'Must be tuple') + + if len(exp_type.members) != len(node.value.elts): + _raise_static_error(node, 'Invalid number of tuple values') + + tuple_data = [ + self.visit_Module_Constant(module, mem.type, arg_node) + for arg_node, mem in zip(node.value.elts, exp_type.members) + if isinstance(arg_node, ast.Constant) + ] + if len(exp_type.members) != len(tuple_data): + _raise_static_error(node, 'Tuple arguments must be constants') + + # Allocate the data + data_block = ModuleDataBlock(tuple_data) + module.data.blocks.append(data_block) + + # Then return the constant as a pointer + return ModuleConstantDef( + node.target.id, + node.lineno, + exp_type, + ConstantTuple(exp_type, tuple_data), + data_block, + ) + + raise NotImplementedError(f'{node} on Module AnnAssign') def visit_Module_stmt(self, module: Module, node: ast.stmt) -> None: if isinstance(node, ast.FunctionDef): @@ -372,7 +406,7 @@ class OurVisitor: if node.id in module.constant_defs: cdef = module.constant_defs[node.id] if exp_type != cdef.type: - _raise_static_error(node, f'Expected {codestyle.type_(exp_type)}, {node.id} is actually {codestyle.type_(act_type)}') + _raise_static_error(node, f'Expected {codestyle.type_(exp_type)}, {node.id} is actually {codestyle.type_(cdef.type)}') return ModuleConstantReference(exp_type, cdef) diff --git a/phasm/stdlib/alloc.py b/phasm/stdlib/alloc.py index a803545..2761bfb 100644 --- a/phasm/stdlib/alloc.py +++ b/phasm/stdlib/alloc.py @@ -12,42 +12,7 @@ ADR_UNALLOC_PTR = ADR_FREE_BLOCK_PTR + 4 UNALLOC_PTR = ADR_UNALLOC_PTR + 4 -@func_wrapper() -def __init__(g: Generator) -> None: - """ - Initializes the memory so we can allocate it - """ - - # Check if the memory is already initialized - g.i32.const(ADR_IDENTIFIER) - g.i32.load() - g.i32.const(IDENTIFIER) - g.i32.eq() - with g.if_(): - # Already initialized, return without any changes - g.return_() - - # Store the first reserved i32 - g.i32.const(ADR_RESERVED0) - g.i32.const(0) - g.i32.store() - - # Store the pointer towards the first free block - # In this case, 0 since we haven't freed any blocks yet - g.i32.const(ADR_FREE_BLOCK_PTR) - g.i32.const(0) - g.i32.store() - - # Store the pointer towards the first unallocated block - # In this case the end of the stdlib.alloc header at the start - g.i32.const(ADR_UNALLOC_PTR) - g.i32.const(UNALLOC_PTR) - g.i32.store() - - # Store that we've initialized the memory - g.i32.const(0) - g.i32.const(IDENTIFIER) - g.i32.store() +# For memory initialization see phasm.compiler.module_data @func_wrapper(exported=False) def __find_free_block__(g: Generator, alloc_size: i32) -> i32: diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 2e6b706..2858472 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -88,8 +88,6 @@ class Suite: # Check if code formatting works assert self.code_py == '\n' + phasm_render(runner.phasm_ast) # \n for formatting in tests - runner.call('stdlib.alloc.__init__') - wasm_args = [] if args: write_header(sys.stderr, 'Memory (pre alloc)') diff --git a/tests/integration/test_constants.py b/tests/integration/test_constants.py index 1593915..36626b1 100644 --- a/tests/integration/test_constants.py +++ b/tests/integration/test_constants.py @@ -3,7 +3,7 @@ import pytest from .helpers import Suite @pytest.mark.integration_test -def test_return(): +def test_i32(): code_py = """ CONSTANT: i32 = 13 @@ -15,3 +15,38 @@ def testEntry() -> i32: result = Suite(code_py).run_code() assert 65 == result.returned_value + +@pytest.mark.integration_test +@pytest.mark.parametrize('type_', ['u8', 'u32', 'u64', ]) +def test_tuple_1(type_): + code_py = f""" +CONSTANT: ({type_}, ) = (65, ) + +@exported +def testEntry() -> {type_}: + return helper(CONSTANT) + +def helper(vector: ({type_}, )) -> {type_}: + return vector[0] +""" + + result = Suite(code_py).run_code() + + assert 65 == result.returned_value + +@pytest.mark.integration_test +def test_tuple_6(): + code_py = """ +CONSTANT: (u8, u8, u32, u32, u64, u64, ) = (11, 22, 3333, 4444, 555555, 666666, ) + +@exported +def testEntry() -> u32: + return helper(CONSTANT) + +def helper(vector: (u8, u8, u32, u32, u64, u64, )) -> u32: + return vector[2] +""" + + result = Suite(code_py).run_code() + + assert 3333 == result.returned_value diff --git a/tests/integration/test_static_checking.py b/tests/integration/test_static_checking.py index 69d0724..ed25023 100644 --- a/tests/integration/test_static_checking.py +++ b/tests/integration/test_static_checking.py @@ -53,3 +53,30 @@ def testEntry() -> (i32, i32, ): with pytest.raises(StaticError, match=f'Static error on line 7: Expected \\(i32, i32, \\), helper actually returns {type_}'): phasm_parse(code_py) + +@pytest.mark.integration_test +def test_tuple_constant_too_few_values(): + code_py = """ +CONSTANT: (u32, u8, u8, ) = (24, 57, ) +""" + + with pytest.raises(StaticError, match=f'Static error on line 2: Invalid number of tuple values'): + phasm_parse(code_py) + +@pytest.mark.integration_test +def test_tuple_constant_too_many_values(): + code_py = """ +CONSTANT: (u32, u8, u8, ) = (24, 57, 1, 1, ) +""" + + with pytest.raises(StaticError, match=f'Static error on line 2: Invalid number of tuple values'): + phasm_parse(code_py) + +@pytest.mark.integration_test +def test_tuple_constant_type_mismatch(): + code_py = """ +CONSTANT: (u32, u8, u8, ) = (24, 4000, 1, ) +""" + + with pytest.raises(StaticError, match=f'Static error on line 2: Integer value out of range; expected 0..255, actual 4000'): + phasm_parse(code_py) diff --git a/tests/integration/test_stdlib_alloc.py b/tests/integration/test_stdlib_alloc.py index a9faf35..da8ccea 100644 --- a/tests/integration/test_stdlib_alloc.py +++ b/tests/integration/test_stdlib_alloc.py @@ -22,51 +22,6 @@ def setup_interpreter(phash_code: str) -> Runner: return runner -@pytest.mark.integration_test -def test___init__(): - code_py = """ -@exported -def testEntry() -> u8: - return 13 -""" - - runner = setup_interpreter(code_py) - - # Garbage in the memory so we can test for it - runner.interpreter_write_memory(0, range(128)) - - write_header(sys.stderr, 'Memory (pre run)') - runner.interpreter_dump_memory(sys.stderr) - - runner.call('stdlib.alloc.__init__') - - write_header(sys.stderr, 'Memory (post run)') - runner.interpreter_dump_memory(sys.stderr) - - assert ( - b'\xC0\xA1\x00\x00' - b'\x00\x00\x00\x00' - b'\x00\x00\x00\x00' - b'\x10\x00\x00\x00' - b'\x10\x11\x12\x13' # Untouched because unused - ) == runner.interpreter_read_memory(0, 20) - -@pytest.mark.integration_test -def test___alloc___no_init(): - code_py = """ -@exported -def testEntry() -> u8: - return 13 -""" - - runner = setup_interpreter(code_py) - - write_header(sys.stderr, 'Memory (pre run)') - runner.interpreter_dump_memory(sys.stderr) - - with pytest.raises(Exception, match='unreachable'): - runner.call('stdlib.alloc.__alloc__', 32) - @pytest.mark.integration_test def test___alloc___ok(): code_py = """ @@ -80,7 +35,6 @@ def testEntry() -> u8: write_header(sys.stderr, 'Memory (pre run)') runner.interpreter_dump_memory(sys.stderr) - runner.call('stdlib.alloc.__init__') offset0 = runner.call('stdlib.alloc.__alloc__', 32) offset1 = runner.call('stdlib.alloc.__alloc__', 32) offset2 = runner.call('stdlib.alloc.__alloc__', 32) From e589223dbb0a4c9bc743bfa6a0122ccc8835aa59 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Thu, 18 Aug 2022 20:53:21 +0200 Subject: [PATCH 70/73] Static Arrays. CRC32 compiles \o/ Doesn't give right answer yet and out of bound check fails. No constructor yet for static arrays, but module constants work. Which don't work yet for tuples and structs. Also, u32 for indexing please. Also, more module constant types. --- examples/buffer.py | 2 +- examples/crc32.html | 47 ++++++ examples/crc32.py | 4 +- examples/index.html | 1 + phasm/codestyle.py | 10 +- phasm/compiler.py | 73 +++++++++- phasm/ourlang.py | 32 +++- phasm/parser.py | 170 +++++++++++++++++----- phasm/typing.py | 24 +++ phasm/wasmgenerator.py | 2 + pylintrc | 5 + tests/integration/test_constants.py | 35 +++++ tests/integration/test_runtime_checks.py | 16 ++ tests/integration/test_simple.py | 40 ++++- tests/integration/test_static_checking.py | 33 ++++- 15 files changed, 448 insertions(+), 46 deletions(-) create mode 100644 examples/crc32.html diff --git a/examples/buffer.py b/examples/buffer.py index 51096c6..6f251ff 100644 --- a/examples/buffer.py +++ b/examples/buffer.py @@ -1,5 +1,5 @@ @exported -def index(inp: bytes, idx: i32) -> i32: +def index(inp: bytes, idx: u32) -> u8: return inp[idx] @exported diff --git a/examples/crc32.html b/examples/crc32.html new file mode 100644 index 0000000..7bd228a --- /dev/null +++ b/examples/crc32.html @@ -0,0 +1,47 @@ + + + +Examples - CRC32 + + +

Buffer

+ +List - Source - WebAssembly + +
+ + + + + + diff --git a/examples/crc32.py b/examples/crc32.py index 781bf21..d531056 100644 --- a/examples/crc32.py +++ b/examples/crc32.py @@ -14,7 +14,7 @@ # } # Might be the wrong table -_CRC32_Table: u32[256] = [ +_CRC32_Table: u32[256] = ( 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, @@ -79,7 +79,7 @@ _CRC32_Table: u32[256] = [ 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351, -] +) def _crc32_f(crc: u32, byt: u8) -> u32: return (crc >> 8) ^ _CRC32_Table[(crc & 0xFF) ^ u32(byt)] diff --git a/examples/index.html b/examples/index.html index f01ade3..96afa50 100644 --- a/examples/index.html +++ b/examples/index.html @@ -6,6 +6,7 @@

Examples

Standard

Technical

diff --git a/phasm/codestyle.py b/phasm/codestyle.py index a09b524..4b38e32 100644 --- a/phasm/codestyle.py +++ b/phasm/codestyle.py @@ -58,6 +58,9 @@ def type_(inp: typing.TypeBase) -> str: return f'({mems}, )' + if isinstance(inp, typing.TypeStaticArray): + return f'{type_(inp.member_type)}[{len(inp.members)}]' + if isinstance(inp, typing.TypeStruct): return inp.name @@ -94,7 +97,7 @@ def expression(inp: ourlang.Expression) -> str: # could not fit in the given float type return str(inp.value) - if isinstance(inp, ourlang.ConstantTuple): + if isinstance(inp, (ourlang.ConstantTuple, ourlang.ConstantStaticArray, )): return '(' + ', '.join( expression(x) for x in inp.value @@ -137,7 +140,10 @@ def expression(inp: ourlang.Expression) -> str: if isinstance(inp, ourlang.AccessStructMember): return f'{expression(inp.varref)}.{inp.member.name}' - if isinstance(inp, ourlang.AccessTupleMember): + if isinstance(inp, (ourlang.AccessTupleMember, ourlang.AccessStaticArrayMember, )): + if isinstance(inp.member, ourlang.Expression): + return f'{expression(inp.varref)}[{expression(inp.member)}]' + return f'{expression(inp.varref)}[{inp.member.idx}]' if isinstance(inp, ourlang.Fold): diff --git a/phasm/compiler.py b/phasm/compiler.py index 1f14aae..3b62065 100644 --- a/phasm/compiler.py +++ b/phasm/compiler.py @@ -62,7 +62,7 @@ def type_(inp: typing.TypeBase) -> wasm.WasmType: if isinstance(inp, typing.TypeFloat64): return wasm.WasmTypeFloat64() - if isinstance(inp, (typing.TypeStruct, typing.TypeTuple, typing.TypeBytes)): + if isinstance(inp, (typing.TypeStruct, typing.TypeTuple, typing.TypeStaticArray, typing.TypeBytes)): # Structs and tuples are passed as pointer # And pointers are i32 return wasm.WasmTypeInt32() @@ -273,6 +273,26 @@ def expression(wgn: WasmGenerator, inp: ourlang.Expression) -> None: wgn.add_statement(f'{mtyp}.load', 'offset=' + str(inp.member.offset)) return + if isinstance(inp, ourlang.AccessStaticArrayMember): + mtyp = LOAD_STORE_TYPE_MAP.get(inp.static_array.member_type.__class__) + if mtyp is None: + # In the future might extend this by having structs or tuples + # as members of static arrays + raise NotImplementedError(expression, inp, inp.member) + + if isinstance(inp.member, typing.TypeStaticArrayMember): + expression(wgn, inp.varref) + wgn.add_statement(f'{mtyp}.load', 'offset=' + str(inp.member.offset)) + return + + expression(wgn, inp.varref) + expression(wgn, inp.member) + wgn.i32.const(inp.static_array.member_type.alloc_size()) + wgn.i32.mul() + wgn.i32.add() + wgn.add_statement(f'{mtyp}.load') + return + if isinstance(inp, ourlang.Fold): expression_fold(wgn, inp) return @@ -285,6 +305,13 @@ def expression(wgn: WasmGenerator, inp: ourlang.Expression) -> None: wgn.i32.const(inp.definition.data_block.address) return + if isinstance(inp.type, typing.TypeStaticArray): + assert isinstance(inp.definition.constant, ourlang.ConstantStaticArray) + assert inp.definition.data_block is not None, 'Combined values are memory stored' + assert inp.definition.data_block.address is not None, 'Value not allocated' + wgn.i32.const(inp.definition.data_block.address) + return + assert inp.definition.data_block is None, 'Primitives are not memory stored' mtyp = LOAD_STORE_TYPE_MAP.get(inp.type.__class__) @@ -471,7 +498,7 @@ def module_data_u32(inp: int) -> bytes: """ Compile: module data, u32 value """ - return struct.pack(' bytes: """ @@ -479,6 +506,30 @@ def module_data_u64(inp: int) -> bytes: """ return struct.pack(' bytes: + """ + Compile: module data, i32 value + """ + return struct.pack(' bytes: + """ + Compile: module data, i64 value + """ + return struct.pack(' bytes: + """ + Compile: module data, f32 value + """ + return struct.pack(' bytes: + """ + Compile: module data, f64 value + """ + return struct.pack(' bytes: """ Compile: module data @@ -505,6 +556,22 @@ def module_data(inp: ourlang.ModuleData) -> bytes: data_list.append(module_data_u64(constant.value)) continue + if isinstance(constant, ourlang.ConstantInt32): + data_list.append(module_data_i32(constant.value)) + continue + + if isinstance(constant, ourlang.ConstantInt64): + data_list.append(module_data_i64(constant.value)) + continue + + if isinstance(constant, ourlang.ConstantFloat32): + data_list.append(module_data_f32(constant.value)) + continue + + if isinstance(constant, ourlang.ConstantFloat64): + data_list.append(module_data_f64(constant.value)) + continue + raise NotImplementedError(constant) block_data = b''.join(data_list) @@ -579,7 +646,7 @@ def _generate_tuple_constructor(wgn: WasmGenerator, inp: ourlang.TupleConstructo wgn.local.get(tmp_var) def _generate_struct_constructor(wgn: WasmGenerator, inp: ourlang.StructConstructor) -> None: - tmp_var = wgn.temp_var_i32('tuple_adr') + tmp_var = wgn.temp_var_i32('struct_adr') # Allocated the required amounts of bytes in memory wgn.i32.const(inp.struct.alloc_size()) diff --git a/phasm/ourlang.py b/phasm/ourlang.py index d7cfffe..5f19d2e 100644 --- a/phasm/ourlang.py +++ b/phasm/ourlang.py @@ -1,7 +1,7 @@ """ Contains the syntax tree for ourlang """ -from typing import Dict, List, Tuple, Optional +from typing import Dict, List, Tuple, Optional, Union import enum @@ -19,6 +19,7 @@ from .typing import ( TypeFloat32, TypeFloat64, TypeBytes, TypeTuple, TypeTupleMember, + TypeStaticArray, TypeStaticArrayMember, TypeStruct, TypeStructMember, ) @@ -135,6 +136,18 @@ class ConstantTuple(Constant): super().__init__(type_) self.value = value +class ConstantStaticArray(Constant): + """ + A StaticArray constant value expression within a statement + """ + __slots__ = ('value', ) + + value: List[Constant] + + def __init__(self, type_: TypeStaticArray, value: List[Constant]) -> None: + super().__init__(type_) + self.value = value + class VariableReference(Expression): """ An variable reference expression within a statement @@ -239,6 +252,23 @@ class AccessTupleMember(Expression): self.varref = varref self.member = member +class AccessStaticArrayMember(Expression): + """ + Access a tuple member for reading of writing + """ + __slots__ = ('varref', 'static_array', 'member', ) + + varref: Union['ModuleConstantReference', VariableReference] + static_array: TypeStaticArray + member: Union[Expression, TypeStaticArrayMember] + + def __init__(self, varref: Union['ModuleConstantReference', VariableReference], static_array: TypeStaticArray, member: Union[TypeStaticArrayMember, Expression], ) -> None: + super().__init__(static_array.member_type) + + self.varref = varref + self.static_array = static_array + self.member = member + class Fold(Expression): """ A (left or right) fold diff --git a/phasm/parser.py b/phasm/parser.py index a9f2f63..d95bfce 100644 --- a/phasm/parser.py +++ b/phasm/parser.py @@ -19,6 +19,8 @@ from .typing import ( TypeStructMember, TypeTuple, TypeTupleMember, + TypeStaticArray, + TypeStaticArrayMember, ) from . import codestyle @@ -30,12 +32,12 @@ from .ourlang import ( Function, Expression, - AccessBytesIndex, AccessStructMember, AccessTupleMember, + AccessBytesIndex, AccessStructMember, AccessTupleMember, AccessStaticArrayMember, BinaryOp, Constant, ConstantFloat32, ConstantFloat64, ConstantInt32, ConstantInt64, ConstantUInt8, ConstantUInt32, ConstantUInt64, - ConstantTuple, + ConstantTuple, ConstantStaticArray, FunctionCall, StructConstructor, TupleConstructor, @@ -249,6 +251,34 @@ class OurVisitor: data_block, ) + if isinstance(exp_type, TypeStaticArray): + if not isinstance(node.value, ast.Tuple): + _raise_static_error(node, 'Must be static array') + + if len(exp_type.members) != len(node.value.elts): + _raise_static_error(node, 'Invalid number of static array values') + + static_array_data = [ + self.visit_Module_Constant(module, exp_type.member_type, arg_node) + for arg_node in node.value.elts + if isinstance(arg_node, ast.Constant) + ] + if len(exp_type.members) != len(static_array_data): + _raise_static_error(node, 'Static array arguments must be constants') + + # Allocate the data + data_block = ModuleDataBlock(static_array_data) + module.data.blocks.append(data_block) + + # Then return the constant as a pointer + return ModuleConstantDef( + node.target.id, + node.lineno, + exp_type, + ConstantStaticArray(exp_type, static_array_data), + data_block, + ) + raise NotImplementedError(f'{node} on Module AnnAssign') def visit_Module_stmt(self, module: Module, node: ast.stmt) -> None: @@ -416,22 +446,22 @@ class OurVisitor: if not isinstance(node.ctx, ast.Load): _raise_static_error(node, 'Must be load context') - if not isinstance(exp_type, TypeTuple): - _raise_static_error(node, f'Expression is expecting a {codestyle.type_(exp_type)}, not a tuple') + if isinstance(exp_type, TypeTuple): + if len(exp_type.members) != len(node.elts): + _raise_static_error(node, f'Expression is expecting a tuple of size {len(exp_type.members)}, but {len(node.elts)} are given') - if len(exp_type.members) != len(node.elts): - _raise_static_error(node, f'Expression is expecting a tuple of size {len(exp_type.members)}, but {len(node.elts)} are given') + tuple_constructor = TupleConstructor(exp_type) - tuple_constructor = TupleConstructor(exp_type) + func = module.functions[tuple_constructor.name] - func = module.functions[tuple_constructor.name] + result = FunctionCall(func) + result.arguments = [ + self.visit_Module_FunctionDef_expr(module, function, our_locals, mem.type, arg_node) + for arg_node, mem in zip(node.elts, exp_type.members) + ] + return result - result = FunctionCall(func) - result.arguments = [ - self.visit_Module_FunctionDef_expr(module, function, our_locals, mem.type, arg_node) - for arg_node, mem in zip(node.elts, exp_type.members) - ] - return result + _raise_static_error(node, f'Expression is expecting a {codestyle.type_(exp_type)}, not a tuple') raise NotImplementedError(f'{node} as expr in FunctionDef') @@ -582,24 +612,37 @@ class OurVisitor: if not isinstance(node.ctx, ast.Load): _raise_static_error(node, 'Must be load context') - if not node.value.id in our_locals: + varref: Union[ModuleConstantReference, VariableReference] + if node.value.id in our_locals: + node_typ = our_locals[node.value.id] + varref = VariableReference(node_typ, node.value.id) + elif node.value.id in module.constant_defs: + constant_def = module.constant_defs[node.value.id] + node_typ = constant_def.type + varref = ModuleConstantReference(node_typ, constant_def) + else: _raise_static_error(node, f'Undefined variable {node.value.id}') - node_typ = our_locals[node.value.id] - slice_expr = self.visit_Module_FunctionDef_expr( - module, function, our_locals, module.types['i32'], node.slice.value, + module, function, our_locals, module.types['u32'], node.slice.value, ) if isinstance(node_typ, TypeBytes): + t_u8 = module.types['u8'] + if exp_type != t_u8: + _raise_static_error(node, f'Expected {codestyle.type_(exp_type)}, {node.value.id}[{codestyle.expression(slice_expr)}] is actually {codestyle.type_(t_u8)}') + + if isinstance(varref, ModuleConstantReference): + raise NotImplementedError(f'{node} from module constant') + return AccessBytesIndex( - module.types['u8'], - VariableReference(node_typ, node.value.id), + t_u8, + varref, slice_expr, ) if isinstance(node_typ, TypeTuple): - if not isinstance(slice_expr, ConstantInt32): + if not isinstance(slice_expr, ConstantUInt32): _raise_static_error(node, 'Must subscript using a constant index') idx = slice_expr.value @@ -607,13 +650,40 @@ class OurVisitor: if len(node_typ.members) <= idx: _raise_static_error(node, f'Index {idx} out of bounds for tuple {node.value.id}') - member = node_typ.members[idx] - if exp_type != member.type: - _raise_static_error(node, f'Expected {codestyle.type_(exp_type)}, {node.value.id}[{idx}] is actually {codestyle.type_(member.type)}') + tuple_member = node_typ.members[idx] + if exp_type != tuple_member.type: + _raise_static_error(node, f'Expected {codestyle.type_(exp_type)}, {node.value.id}[{idx}] is actually {codestyle.type_(tuple_member.type)}') + + if isinstance(varref, ModuleConstantReference): + raise NotImplementedError(f'{node} from module constant') return AccessTupleMember( - VariableReference(node_typ, node.value.id), - member, + varref, + tuple_member, + ) + + if isinstance(node_typ, TypeStaticArray): + if exp_type != node_typ.member_type: + _raise_static_error(node, f'Expected {codestyle.type_(exp_type)}, {node.value.id}[{idx}] is actually {codestyle.type_(node_typ.member_type)}') + + if not isinstance(slice_expr, ConstantInt32): + return AccessStaticArrayMember( + varref, + node_typ, + slice_expr, + ) + + idx = slice_expr.value + + if len(node_typ.members) <= idx: + _raise_static_error(node, f'Index {idx} out of bounds for static array {node.value.id}') + + static_array_member = node_typ.members[idx] + + return AccessStaticArrayMember( + varref, + node_typ, + static_array_member, ) _raise_static_error(node, f'Cannot take index of {node_typ} {node.value.id}') @@ -705,25 +775,59 @@ class OurVisitor: _raise_static_error(node, f'Unrecognized type {node.id}') + if isinstance(node, ast.Subscript): + if not isinstance(node.value, ast.Name): + _raise_static_error(node, 'Must be name') + if not isinstance(node.slice, ast.Index): + _raise_static_error(node, 'Must subscript using an index') + if not isinstance(node.slice.value, ast.Constant): + _raise_static_error(node, 'Must subscript using a constant index') + if not isinstance(node.slice.value.value, int): + _raise_static_error(node, 'Must subscript using a constant integer index') + if not isinstance(node.ctx, ast.Load): + _raise_static_error(node, 'Must be load context') + + if node.value.id in module.types: + member_type = module.types[node.value.id] + else: + _raise_static_error(node, f'Unrecognized type {node.value.id}') + + type_static_array = TypeStaticArray(member_type) + + offset = 0 + + for idx in range(node.slice.value.value): + static_array_member = TypeStaticArrayMember(idx, offset) + + type_static_array.members.append(static_array_member) + offset += member_type.alloc_size() + + key = f'{node.value.id}[{node.slice.value.value}]' + + if key not in module.types: + module.types[key] = type_static_array + + return module.types[key] + if isinstance(node, ast.Tuple): if not isinstance(node.ctx, ast.Load): _raise_static_error(node, 'Must be load context') - result = TypeTuple() + type_tuple = TypeTuple() offset = 0 for idx, elt in enumerate(node.elts): - member = TypeTupleMember(idx, self.visit_type(module, elt), offset) + tuple_member = TypeTupleMember(idx, self.visit_type(module, elt), offset) - result.members.append(member) - offset += member.type.alloc_size() + type_tuple.members.append(tuple_member) + offset += tuple_member.type.alloc_size() - key = result.render_internal_name() + key = type_tuple.render_internal_name() if key not in module.types: - module.types[key] = result - constructor = TupleConstructor(result) + module.types[key] = type_tuple + constructor = TupleConstructor(type_tuple) module.functions[constructor.name] = constructor return module.types[key] diff --git a/phasm/typing.py b/phasm/typing.py index 0a29f5f..e56f7a9 100644 --- a/phasm/typing.py +++ b/phasm/typing.py @@ -137,6 +137,30 @@ class TypeTuple(TypeBase): for x in self.members ) +class TypeStaticArrayMember: + """ + Represents a static array member + """ + def __init__(self, idx: int, offset: int) -> None: + self.idx = idx + self.offset = offset + +class TypeStaticArray(TypeBase): + """ + The static array type + """ + __slots__ = ('member_type', 'members', ) + + member_type: TypeBase + members: List[TypeStaticArrayMember] + + def __init__(self, member_type: TypeBase) -> None: + self.member_type = member_type + self.members = [] + + def alloc_size(self) -> int: + return self.member_type.alloc_size() * len(self.members) + class TypeStructMember: """ Represents a struct member diff --git a/phasm/wasmgenerator.py b/phasm/wasmgenerator.py index 57b1fe5..48cca1a 100644 --- a/phasm/wasmgenerator.py +++ b/phasm/wasmgenerator.py @@ -30,6 +30,8 @@ class Generator_i32i64: # 2.4.1. Numeric Instructions # ibinop self.add = functools.partial(self.generator.add_statement, f'{prefix}.add') + self.sub = functools.partial(self.generator.add_statement, f'{prefix}.sub') + self.mul = functools.partial(self.generator.add_statement, f'{prefix}.mul') # irelop self.eq = functools.partial(self.generator.add_statement, f'{prefix}.eq') diff --git a/pylintrc b/pylintrc index b38e5c0..0591be3 100644 --- a/pylintrc +++ b/pylintrc @@ -1,5 +1,10 @@ [MASTER] disable=C0122,R0903,R0911,R0912,R0913,R0915,R1710,W0223 +max-line-length=180 + [stdlib] good-names=g + +[tests] +disable=C0116, diff --git a/tests/integration/test_constants.py b/tests/integration/test_constants.py index 36626b1..19f0203 100644 --- a/tests/integration/test_constants.py +++ b/tests/integration/test_constants.py @@ -50,3 +50,38 @@ def helper(vector: (u8, u8, u32, u32, u64, u64, )) -> u32: result = Suite(code_py).run_code() assert 3333 == result.returned_value + +@pytest.mark.integration_test +@pytest.mark.parametrize('type_', ['u8', 'u32', 'u64', ]) +def test_static_array_1(type_): + code_py = f""" +CONSTANT: {type_}[1] = (65, ) + +@exported +def testEntry() -> {type_}: + return helper(CONSTANT) + +def helper(vector: {type_}[1]) -> {type_}: + return vector[0] +""" + + result = Suite(code_py).run_code() + + assert 65 == result.returned_value + +@pytest.mark.integration_test +def test_static_array_6(): + code_py = """ +CONSTANT: u32[6] = (11, 22, 3333, 4444, 555555, 666666, ) + +@exported +def testEntry() -> u32: + return helper(CONSTANT) + +def helper(vector: u32[6]) -> u32: + return vector[2] +""" + + result = Suite(code_py).run_code() + + assert 3333 == result.returned_value diff --git a/tests/integration/test_runtime_checks.py b/tests/integration/test_runtime_checks.py index 6a70032..97d6542 100644 --- a/tests/integration/test_runtime_checks.py +++ b/tests/integration/test_runtime_checks.py @@ -13,3 +13,19 @@ def testEntry(f: bytes) -> u8: result = Suite(code_py).run_code(b'Short', b'Long' * 100) assert 0 == result.returned_value + +@pytest.mark.integration_test +def test_static_array_index_out_of_bounds(): + code_py = """ +CONSTANT0: u32[3] = (24, 57, 80, ) + +CONSTANT1: u32[16] = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, ) + +@exported +def testEntry() -> u32: + return CONSTANT0[16] +""" + + result = Suite(code_py).run_code() + + assert 0 == result.returned_value diff --git a/tests/integration/test_simple.py b/tests/integration/test_simple.py index b6bfbc5..f0c2993 100644 --- a/tests/integration/test_simple.py +++ b/tests/integration/test_simple.py @@ -427,7 +427,7 @@ def helper(shape1: Rectangle, shape2: Rectangle) -> i32: @pytest.mark.integration_test @pytest.mark.parametrize('type_', COMPLETE_SIMPLE_TYPES) -def test_tuple_simple(type_): +def test_tuple_simple_constructor(type_): code_py = f""" @exported def testEntry() -> {type_}: @@ -457,6 +457,44 @@ def helper(v: (f32, f32, f32, )) -> f32: assert 3.74 < result.returned_value < 3.75 +@pytest.mark.integration_test +@pytest.mark.parametrize('type_', COMPLETE_SIMPLE_TYPES) +def test_static_array_module_constant(type_): + code_py = f""" +CONSTANT: {type_}[3] = (24, 57, 80, ) + +@exported +def testEntry() -> {type_}: + return helper(CONSTANT) + +def helper(array: {type_}[3]) -> {type_}: + return array[0] + array[1] + array[2] +""" + + result = Suite(code_py).run_code() + + assert 161 == result.returned_value + assert TYPE_MAP[type_] == type(result.returned_value) + +@pytest.mark.integration_test +@pytest.mark.parametrize('type_', COMPLETE_SIMPLE_TYPES) +def test_static_array_indexed(type_): + code_py = f""" +CONSTANT: {type_}[3] = (24, 57, 80, ) + +@exported +def testEntry() -> {type_}: + return helper(CONSTANT, 0, 1, 2) + +def helper(array: {type_}[3], i0: u32, i1: u32, i2: u32) -> {type_}: + return array[i0] + array[i1] + array[i2] +""" + + result = Suite(code_py).run_code() + + assert 161 == result.returned_value + assert TYPE_MAP[type_] == type(result.returned_value) + @pytest.mark.integration_test def test_bytes_address(): code_py = """ diff --git a/tests/integration/test_static_checking.py b/tests/integration/test_static_checking.py index ed25023..1544537 100644 --- a/tests/integration/test_static_checking.py +++ b/tests/integration/test_static_checking.py @@ -60,7 +60,7 @@ def test_tuple_constant_too_few_values(): CONSTANT: (u32, u8, u8, ) = (24, 57, ) """ - with pytest.raises(StaticError, match=f'Static error on line 2: Invalid number of tuple values'): + with pytest.raises(StaticError, match='Static error on line 2: Invalid number of tuple values'): phasm_parse(code_py) @pytest.mark.integration_test @@ -69,7 +69,7 @@ def test_tuple_constant_too_many_values(): CONSTANT: (u32, u8, u8, ) = (24, 57, 1, 1, ) """ - with pytest.raises(StaticError, match=f'Static error on line 2: Invalid number of tuple values'): + with pytest.raises(StaticError, match='Static error on line 2: Invalid number of tuple values'): phasm_parse(code_py) @pytest.mark.integration_test @@ -78,5 +78,32 @@ def test_tuple_constant_type_mismatch(): CONSTANT: (u32, u8, u8, ) = (24, 4000, 1, ) """ - with pytest.raises(StaticError, match=f'Static error on line 2: Integer value out of range; expected 0..255, actual 4000'): + with pytest.raises(StaticError, match='Static error on line 2: Integer value out of range; expected 0..255, actual 4000'): + phasm_parse(code_py) + +@pytest.mark.integration_test +def test_static_array_constant_too_few_values(): + code_py = """ +CONSTANT: u8[3] = (24, 57, ) +""" + + with pytest.raises(StaticError, match='Static error on line 2: Invalid number of static array values'): + phasm_parse(code_py) + +@pytest.mark.integration_test +def test_static_array_constant_too_many_values(): + code_py = """ +CONSTANT: u8[3] = (24, 57, 1, 1, ) +""" + + with pytest.raises(StaticError, match='Static error on line 2: Invalid number of static array values'): + phasm_parse(code_py) + +@pytest.mark.integration_test +def test_static_array_constant_type_mismatch(): + code_py = """ +CONSTANT: u8[3] = (24, 4000, 1, ) +""" + + with pytest.raises(StaticError, match='Static error on line 2: Integer value out of range; expected 0..255, actual 4000'): phasm_parse(code_py) From 98167cfdec85d8cae8b12b9a05cbebcdc3650c21 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 20 Aug 2022 18:00:20 +0200 Subject: [PATCH 71/73] Made tests more consistent. Implemented CRC32 test. --- .gitignore | 1 + examples/buffer.html | 49 ++++++++++----------- examples/crc32.html | 89 ++++++++++++++++++++++++++++---------- examples/crc32.py | 97 ++++++++++++++---------------------------- examples/fib.html | 41 +++++++++++++----- examples/fold.html | 62 +++++++++++---------------- examples/imported.html | 44 +++++++++++++------ examples/imported.py | 2 +- examples/include.js | 64 ++++++++++++++++++++++++++++ 9 files changed, 275 insertions(+), 174 deletions(-) create mode 100644 examples/include.js diff --git a/.gitignore b/.gitignore index 7f7c00b..b8a823a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /*.wasm /*.wat +/.coverage /venv __pycache__ diff --git a/examples/buffer.html b/examples/buffer.html index cf4e442..a9e46ee 100644 --- a/examples/buffer.html +++ b/examples/buffer.html @@ -11,38 +11,39 @@
+ diff --git a/examples/crc32.html b/examples/crc32.html index 7bd228a..d867f5b 100644 --- a/examples/crc32.html +++ b/examples/crc32.html @@ -11,35 +11,80 @@
+ diff --git a/examples/crc32.py b/examples/crc32.py index d531056..294b3c5 100644 --- a/examples/crc32.py +++ b/examples/crc32.py @@ -13,72 +13,39 @@ # return crc32; # } -# Might be the wrong table _CRC32_Table: u32[256] = ( - 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, - 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, - 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, - 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24, - 0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B, - 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384, - 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54, - 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B, - 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, - 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, - 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5, - 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA, - 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, - 0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A, - 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, - 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595, - 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, - 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957, - 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, - 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198, - 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927, - 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, - 0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8, - 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7, - 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096, - 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789, - 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, - 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, - 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, - 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6, - 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, - 0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829, - 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, - 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93, - 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, - 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C, - 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, - 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC, - 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, - 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, - 0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652, - 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D, - 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D, - 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982, - 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, - 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622, - 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, - 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED, - 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, - 0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F, - 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, - 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0, - 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F, - 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540, - 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, - 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F, - 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE, - 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1, - 0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321, - 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E, - 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, - 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, - 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, - 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351, + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, + 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, + 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, + 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, + 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, + 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, + 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F, + 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, + 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, + 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, + 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, + 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, + 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, + 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, + 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, + 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, + 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, + 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, + 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, + 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, + 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, + 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, + 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, + 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, + 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, + 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, + 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, + 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, + 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, + 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, + 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, + 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D, ) def _crc32_f(crc: u32, byt: u8) -> u32: diff --git a/examples/fib.html b/examples/fib.html index 7a2bccb..35c6109 100644 --- a/examples/fib.html +++ b/examples/fib.html @@ -10,23 +10,44 @@
+ diff --git a/examples/fold.html b/examples/fold.html index d47e580..0a9f93f 100644 --- a/examples/fold.html +++ b/examples/fold.html @@ -10,51 +10,37 @@
- + diff --git a/examples/imported.html b/examples/imported.html index a69c02e..b0a6957 100644 --- a/examples/imported.html +++ b/examples/imported.html @@ -10,7 +10,7 @@
- + diff --git a/examples/imported.py b/examples/imported.py index f5c2663..4083986 100644 --- a/examples/imported.py +++ b/examples/imported.py @@ -3,5 +3,5 @@ def log(no: i32) -> None: pass @exported -def run(a: i32, b: i32) -> None: +def mult_and_log(a: i32, b: i32) -> None: return log(a * b) diff --git a/examples/include.js b/examples/include.js new file mode 100644 index 0000000..bd93dcf --- /dev/null +++ b/examples/include.js @@ -0,0 +1,64 @@ +function alloc_bytes(app, data) +{ + let stdlib_types___alloc_bytes__ = app.instance.exports['stdlib.types.__alloc_bytes__'] + + if( typeof data == 'string' ) { + // TODO: Unicode + data = Uint8Array.from(data.split('').map(x => x.charCodeAt())); + } + + let offset = stdlib_types___alloc_bytes__(data.length); + let i8arr = new Uint8Array(app.instance.exports.memory.buffer, offset + 4, data.length); + i8arr.set(data); + + return offset; +} + +function run_times(times, callback) +{ + let sum = 0; + for(let idx = 0; idx < times; idx += 1) { + const t0 = performance.now(); + callback(); + const t1 = performance.now(); + sum += t1 - t0; + } + console.log(sum); + return sum / times; +} + +function test_result(is_pass, data) +{ + data = data || {}; + + let result_details = document.createElement('details'); + + let result_summary = document.createElement('summary'); + result_summary.textContent = + (is_pass ? 'Test passed: ' : 'Test failed: ') + + (data.summary ?? '(no summary)') + ; + result_summary.setAttribute('style', is_pass ? 'background: green' : 'background: red'); + result_details.appendChild(result_summary); + + if( data.attributes ) { + let table = document.createElement('table'); + + Object.keys(data.attributes).forEach(idx => { + let td0 = document.createElement('td'); + td0.textContent = idx; + let td1 = document.createElement('td'); + td1.textContent = data.attributes[idx]; + + let tr = document.createElement('tr'); + tr.appendChild(td0); + tr.appendChild(td1); + + table.appendChild(tr); + }); + result_details.append(table); + } + + let results = document.getElementById('results'); + results.appendChild(result_details); +} From 7a8b1baa25c5f0bef8088195daf31b5041acc7aa Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sat, 20 Aug 2022 18:14:28 +0200 Subject: [PATCH 72/73] Some repo cleanup --- Makefile | 2 +- README.md | 100 +++++++++++++++++++++++++++---- TODO.md | 2 + examples/fib.py | 6 +- tests/integration/helpers.py | 38 ------------ tests/integration/test_helper.py | 2 +- 6 files changed, 96 insertions(+), 54 deletions(-) diff --git a/Makefile b/Makefile index 8f4d045..ad186c2 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ examples: venv/.done $(subst .py,.wasm,$(wildcard examples/*.py)) $(subst .py,.w venv/bin/python3 -m http.server --directory examples test: venv/.done - WAT2WASM=$(WAT2WASM) venv/bin/pytest tests $(TEST_FLAGS) + venv/bin/pytest tests $(TEST_FLAGS) lint: venv/.done venv/bin/pylint phasm diff --git a/README.md b/README.md index 5ab49f7..c9cb056 100644 --- a/README.md +++ b/README.md @@ -3,19 +3,97 @@ phasm Elevator pitch -------------- -A language that looks like Python, handles like Haskell, and compiles to -WebAssembly. +A programming language, that looks like Python, handles like Haskell, +and compiles directly to WebAssembly. -Naming ------- +Project state +------------- +This is a hobby project for now. Use at your own risk. + +How to run +---------- +You should only need make and python3. Currently, we're working with python3.8, +since we're using the python ast parser, it might not work on other versions. + +To run the examples: +```sh +make examples +``` + +To run the tests: +```sh +make test +``` + +To run the linting and type checking: +```sh +make lint typecheck +``` + +To compile a Phasm file: +```sh +python3.8 -m phasm source.py output.wat +``` + +Additional required tools +------------------------- +At the moment, the compiler outputs WebAssembly text format. To actually +get a binary, you will need the wat2wasm tool[6]. + +Example +------- +For more examples, see the examples directory. +```py +def helper(n: u64, a: u64, b: u64) -> u64: + if n < 1: + return a + b + + return helper(n - 1, a + b, a) + +@exported +def fib(n: u64) -> u64: + if n == 0: + return 0 + + if n == 1: + return 1 + + return helper(n - 1, 0, 1) +``` + +Gotcha's +-------- +- When importing and exporting unsigned values to WebAssembly, they will become + signed, as WebAssembly has no native unsigned type. You may need to cast + or reinterpret them. +- Currently, Phasm files have the .py extension, which helps with syntax + highlighting, that might change in the future. + +Contributing +------------ +At this time, we're mostly looking for use cases for WebAssembly, other than to +compile existing C code and running them in the browser. The goal of WebAssembly +is to enable high-performance applications on web pages[5]. Though most people +seem to use it to have existing code run in the browser. + +If you have a situation where WebAssembly would be useful for it's speed, we're +interested to see what you want to use it for. + +Also, if you are trying out Phasm, and you're running into a limitation, we're +interested in a minimal test case that shows what you want to achieve and how +Phasm currently fails you. + +Name origin +----------- - p from python - ha from Haskell - asm from WebAssembly -You will need wat2wasm from github.com/WebAssembly/wabt in your path. - -Ideas -===== -- https://github.com/wasmerio/wasmer-python -- https://github.com/diekmann/wasm-fizzbuzz -- https://blog.scottlogic.com/2018/04/26/webassembly-by-hand.html +References +---------- +[1] https://www.python.org/ +[2] https://www.haskell.org/ +[3] https://webassembly.org/ +[4] https://www.w3.org/TR/wasm-core-1/ +[5] https://en.wikipedia.org/w/index.php?title=WebAssembly&oldid=1103639883 +[6] https://github.com/WebAssembly/wabt diff --git a/TODO.md b/TODO.md index 41ae3da..0da4619 100644 --- a/TODO.md +++ b/TODO.md @@ -4,3 +4,5 @@ - Implement a proper type matching / checking system - Check if we can use DataView in the Javascript examples, e.g. with setUint32 - Storing u8 in memory still claims 32 bits (since that's what you need in local variables). However, using load8_u / loadu_s we can optimize this. +- Implement a FizzBuzz example +- Also, check the codes for FIXME and TODO diff --git a/examples/fib.py b/examples/fib.py index 8536245..af435c1 100644 --- a/examples/fib.py +++ b/examples/fib.py @@ -1,11 +1,11 @@ -def helper(n: i64, a: i64, b: i64) -> i64: +def helper(n: u64, a: u64, b: u64) -> u64: if n < 1: return a + b return helper(n - 1, a + b, a) @exported -def fib(n: i64) -> i64: +def fib(n: u64) -> u64: if n == 0: return 0 @@ -15,5 +15,5 @@ def fib(n: i64) -> i64: return helper(n - 1, 0, 1) @exported -def testEntry() -> i64: +def testEntry() -> u64: return fib(40) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 2858472..ca4c8a7 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -1,49 +1,11 @@ -import io -import os -import subprocess import sys -from tempfile import NamedTemporaryFile - -import pywasm - -import wasm3 - -import wasmer -import wasmer_compiler_cranelift - -import wasmtime - from phasm.codestyle import phasm_render -from phasm.compiler import phasm_compile -from phasm.parser import phasm_parse from . import runners DASHES = '-' * 16 -def wat2wasm(code_wat): - path = os.environ.get('WAT2WASM', 'wat2wasm') - - with NamedTemporaryFile('w+t') as input_fp: - input_fp.write(code_wat) - input_fp.flush() - - with NamedTemporaryFile('w+b') as output_fp: - subprocess.run( - [ - path, - input_fp.name, - '-o', - output_fp.name, - ], - check=True, - ) - - output_fp.seek(0) - - return output_fp.read() - class SuiteResult: def __init__(self): self.returned_value = None diff --git a/tests/integration/test_helper.py b/tests/integration/test_helper.py index 387eb5a..cb44021 100644 --- a/tests/integration/test_helper.py +++ b/tests/integration/test_helper.py @@ -5,7 +5,7 @@ import pytest from pywasm import binary from pywasm import Runtime -from .helpers import wat2wasm +from wasmer import wat2wasm def run(code_wat): code_wasm = wat2wasm(code_wat) From f683961af8ce38e6f179e70e54322984dc19b543 Mon Sep 17 00:00:00 2001 From: "Johan B.W. de Vries" Date: Sun, 21 Aug 2022 14:57:43 +0200 Subject: [PATCH 73/73] Timing results Looks like WebAssembly in Chromium is about 35% faster, but the Javascript engine in Firefox is another 59% faster --- examples/crc32.html | 145 +++++++++++++++++++++++++++++++++++++++----- examples/include.js | 74 ++++++++++++++++------ 2 files changed, 183 insertions(+), 36 deletions(-) diff --git a/examples/crc32.html b/examples/crc32.html index d867f5b..949dadf 100644 --- a/examples/crc32.html +++ b/examples/crc32.html @@ -6,10 +6,84 @@

Buffer

-List - Source - WebAssembly - +List - Source - WebAssembly
+
+Note: This tests performs some timing comparison, please wait a few seconds for the results.
+

Measurement log

+

AMD Ryzen 7 3700X 8-Core, Ubuntu 20.04, Linux 5.4.0-124-generic

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TestInterpreterSetupWebAssemblyJavascript
Lynx * 65536Chromium 104.0.5112.101DevTools closed9.3512.56
Lynx * 65536Chromium 104.0.5112.101DevTools open14.7112.72
Lynx * 65536Chromium 104.0.5112.101Record page load9.4412.69
Lynx * 65536Firefox 103DevTools closed9.025.86
Lynx * 65536Firefox 103DevTools open9.015.83
Lynx * 65536Firefox 103Record page load72.415.85
Lynx * 1048576Chromium 104.0.5112.101DevTools closed149.24202.36
Lynx * 1048576Firefox 103DevTools closed145.0191.44
+ +Notes:
+- Firefox seems faster than Chromium in my setup for Javascript, WebAssembly seems about the same.
+- Having DevTools open in Chromium seems to slow down the WebAssembly by about 30%, but not when doing a recording of the page load.
+- WebAssembly in Firefox seems to slow down when doing a recording of the page load, which makes sense, but the Javascript does not.
diff --git a/examples/include.js b/examples/include.js index bd93dcf..50f9324 100644 --- a/examples/include.js +++ b/examples/include.js @@ -14,17 +14,33 @@ function alloc_bytes(app, data) return offset; } -function run_times(times, callback) +function run_times(times, callback, tweak) { let sum = 0; + let max = 0; + let min = 1000000000000000000; + let values = []; for(let idx = 0; idx < times; idx += 1) { + if( tweak ) { + tweak(); + } + const t0 = performance.now(); - callback(); + let result = callback(); const t1 = performance.now(); - sum += t1 - t0; + let time = t1 - t0; + sum += time; + values.push({'time': time, 'result': result}); + max = max < time ? time : max; + min = min > time ? time : min; + } + return { + 'min': min, + 'avg': sum / times, + 'max': max, + 'sum': sum, + 'values': values, } - console.log(sum); - return sum / times; } function test_result(is_pass, data) @@ -42,23 +58,41 @@ function test_result(is_pass, data) result_details.appendChild(result_summary); if( data.attributes ) { - let table = document.createElement('table'); - - Object.keys(data.attributes).forEach(idx => { - let td0 = document.createElement('td'); - td0.textContent = idx; - let td1 = document.createElement('td'); - td1.textContent = data.attributes[idx]; - - let tr = document.createElement('tr'); - tr.appendChild(td0); - tr.appendChild(td1); - - table.appendChild(tr); - }); - result_details.append(table); + result_table(data, result_details); } let results = document.getElementById('results'); results.appendChild(result_details); } + +function result_table(attributes, parent) +{ + let table = document.createElement('table'); + + Object.keys(attributes).forEach(idx => { + let td0 = document.createElement('td'); + td0.setAttribute('style', 'vertical-align: top;'); + td0.textContent = idx; + let td1 = document.createElement('td'); + if( typeof(attributes[idx]) == 'object' ) { + let result_details = document.createElement('details'); + + let result_summary = document.createElement('summary'); + result_summary.textContent = 'Show me'; + result_details.appendChild(result_summary); + + result_table(attributes[idx], result_details); + + td1.appendChild(result_details); + } else { + td1.textContent = attributes[idx]; + } + + let tr = document.createElement('tr'); + tr.appendChild(td0); + tr.appendChild(td1); + + table.appendChild(tr); + }); + parent.append(table); +}