MVP #1

Merged
jbwdevries merged 73 commits from idea_crc32 into master 2022-08-21 12:59:21 +00:00
7 changed files with 256 additions and 118 deletions
Showing only changes of commit 0cf8a246fe - Show all commits

View File

@ -2,3 +2,4 @@
- Implement foldl for bytes
- Implement a trace() builtin for debugging
- Implement a proper type matching / checking system

View File

@ -1,17 +1,14 @@
"""
This module contains the code to convert parsed Ourlang into WebAssembly code
"""
from typing import Generator
from . import codestyle
from . import ourlang
from . import typing
from . import wasm
from .stdlib import alloc as stdlib_alloc
from .stdlib import types as stdlib_types
from .wasmeasy import i32, i64
Statements = Generator[wasm.Statement, None, None]
from .wasmgenerator import Generator as WasmGenerator
LOAD_STORE_TYPE_MAP = {
typing.TypeUInt8: 'i32',
@ -115,123 +112,123 @@ I64_OPERATOR_MAP = {
'>=': 'ge_s',
}
def expression(inp: ourlang.Expression) -> Statements:
def expression(wgn: WasmGenerator, inp: ourlang.Expression) -> None:
"""
Compile: Any expression
"""
if isinstance(inp, ourlang.ConstantUInt8):
yield i32.const(inp.value)
wgn.i32.const(inp.value)
return
if isinstance(inp, ourlang.ConstantUInt32):
yield i32.const(inp.value)
wgn.i32.const(inp.value)
return
if isinstance(inp, ourlang.ConstantUInt64):
yield i64.const(inp.value)
wgn.i64.const(inp.value)
return
if isinstance(inp, ourlang.ConstantInt32):
yield i32.const(inp.value)
wgn.i32.const(inp.value)
return
if isinstance(inp, ourlang.ConstantInt64):
yield i64.const(inp.value)
wgn.i64.const(inp.value)
return
if isinstance(inp, ourlang.ConstantFloat32):
yield wasm.Statement('f32.const', str(inp.value))
wgn.f32.const(inp.value)
return
if isinstance(inp, ourlang.ConstantFloat64):
yield wasm.Statement('f64.const', str(inp.value))
wgn.f64.const(inp.value)
return
if isinstance(inp, ourlang.VariableReference):
yield wasm.Statement('local.get', '${}'.format(inp.name))
wgn.add_statement('local.get', '${}'.format(inp.name))
return
if isinstance(inp, ourlang.BinaryOp):
yield from expression(inp.left)
yield from expression(inp.right)
expression(wgn, inp.left)
expression(wgn, inp.right)
if isinstance(inp.type, typing.TypeUInt8):
if operator := U8_OPERATOR_MAP.get(inp.operator, None):
yield wasm.Statement(f'i32.{operator}')
wgn.add_statement(f'i32.{operator}')
return
if isinstance(inp.type, typing.TypeUInt32):
if operator := OPERATOR_MAP.get(inp.operator, None):
yield wasm.Statement(f'i32.{operator}')
wgn.add_statement(f'i32.{operator}')
return
if operator := U32_OPERATOR_MAP.get(inp.operator, None):
yield wasm.Statement(f'i32.{operator}')
wgn.add_statement(f'i32.{operator}')
return
if isinstance(inp.type, typing.TypeUInt64):
if operator := OPERATOR_MAP.get(inp.operator, None):
yield wasm.Statement(f'i64.{operator}')
wgn.add_statement(f'i64.{operator}')
return
if operator := U64_OPERATOR_MAP.get(inp.operator, None):
yield wasm.Statement(f'i64.{operator}')
wgn.add_statement(f'i64.{operator}')
return
if isinstance(inp.type, typing.TypeInt32):
if operator := OPERATOR_MAP.get(inp.operator, None):
yield wasm.Statement(f'i32.{operator}')
wgn.add_statement(f'i32.{operator}')
return
if operator := I32_OPERATOR_MAP.get(inp.operator, None):
yield wasm.Statement(f'i32.{operator}')
wgn.add_statement(f'i32.{operator}')
return
if isinstance(inp.type, typing.TypeInt64):
if operator := OPERATOR_MAP.get(inp.operator, None):
yield wasm.Statement(f'i64.{operator}')
wgn.add_statement(f'i64.{operator}')
return
if operator := I64_OPERATOR_MAP.get(inp.operator, None):
yield wasm.Statement(f'i64.{operator}')
wgn.add_statement(f'i64.{operator}')
return
if isinstance(inp.type, typing.TypeFloat32):
if operator := OPERATOR_MAP.get(inp.operator, None):
yield wasm.Statement(f'f32.{operator}')
wgn.add_statement(f'f32.{operator}')
return
if isinstance(inp.type, typing.TypeFloat64):
if operator := OPERATOR_MAP.get(inp.operator, None):
yield wasm.Statement(f'f64.{operator}')
wgn.add_statement(f'f64.{operator}')
return
raise NotImplementedError(expression, inp.type, inp.operator)
if isinstance(inp, ourlang.UnaryOp):
yield from expression(inp.right)
expression(wgn, inp.right)
if isinstance(inp.type, typing.TypeFloat32):
if inp.operator in ourlang.WEBASSEMBLY_BUILDIN_FLOAT_OPS:
yield wasm.Statement(f'f32.{inp.operator}')
wgn.add_statement(f'f32.{inp.operator}')
return
if isinstance(inp.type, typing.TypeFloat64):
if inp.operator in ourlang.WEBASSEMBLY_BUILDIN_FLOAT_OPS:
yield wasm.Statement(f'f64.{inp.operator}')
wgn.add_statement(f'f64.{inp.operator}')
return
if isinstance(inp.type, typing.TypeInt32):
if inp.operator == 'len':
if isinstance(inp.right.type, typing.TypeBytes):
yield i32.load()
wgn.i32.load()
return
raise NotImplementedError(expression, inp.type, inp.operator)
if isinstance(inp, ourlang.FunctionCall):
for arg in inp.arguments:
yield from expression(arg)
expression(wgn, arg)
yield wasm.Statement('call', '${}'.format(inp.function.name))
wgn.add_statement('call', '${}'.format(inp.function.name))
return
if isinstance(inp, ourlang.AccessBytesIndex):
if not isinstance(inp.type, typing.TypeUInt8):
raise NotImplementedError(inp, inp.type)
yield from expression(inp.varref)
yield from expression(inp.index)
yield wasm.Statement('call', '$stdlib.types.__subscript_bytes__')
expression(wgn, inp.varref)
expression(wgn, inp.index)
wgn.call(stdlib_types.__subscript_bytes__)
return
if isinstance(inp, ourlang.AccessStructMember):
@ -241,8 +238,8 @@ def expression(inp: ourlang.Expression) -> Statements:
# as members of struct or tuples
raise NotImplementedError(expression, inp, inp.member)
yield from expression(inp.varref)
yield wasm.Statement(f'{mtyp}.load', 'offset=' + str(inp.member.offset))
expression(wgn, inp.varref)
wgn.add_statement(f'{mtyp}.load', 'offset=' + str(inp.member.offset))
return
if isinstance(inp, ourlang.AccessTupleMember):
@ -252,47 +249,62 @@ def expression(inp: ourlang.Expression) -> Statements:
# as members of struct or tuples
raise NotImplementedError(expression, inp, inp.member)
yield from expression(inp.varref)
yield wasm.Statement(f'{mtyp}.load', 'offset=' + str(inp.member.offset))
expression(wgn, inp.varref)
wgn.add_statement(f'{mtyp}.load', 'offset=' + str(inp.member.offset))
return
if isinstance(inp, ourlang.Fold):
mtyp = LOAD_STORE_TYPE_MAP.get(inp.base.type.__class__)
if mtyp is None:
# In the future might extend this by having structs or tuples
# as members of struct or tuples
raise NotImplementedError(expression, inp, inp.base)
if inp.iter.type.__class__.__name__ != 'TypeBytes':
raise NotImplementedError(expression, inp, inp.iter.type)
tmp_var = wgn.temp_var_u8(f'fold_{codestyle.type_(inp.type)}')
expression(wgn, inp.base)
wgn.local.set(tmp_var)
# FIXME: Do a loop
wgn.unreachable(comment='Not implemented yet')
return
raise NotImplementedError(expression, inp)
def statement_return(inp: ourlang.StatementReturn) -> Statements:
def statement_return(wgn: WasmGenerator, inp: ourlang.StatementReturn) -> None:
"""
Compile: Return statement
"""
yield from expression(inp.value)
yield wasm.Statement('return')
expression(wgn, inp.value)
wgn.return_()
def statement_if(inp: ourlang.StatementIf) -> Statements:
def statement_if(wgn: WasmGenerator, inp: ourlang.StatementIf) -> None:
"""
Compile: If statement
"""
yield from expression(inp.test)
yield wasm.Statement('if')
expression(wgn, inp.test)
with wgn.if_():
for stat in inp.statements:
yield from statement(stat)
statement(wgn, stat)
if inp.else_statements:
yield wasm.Statement('else')
for stat in inp.else_statements:
yield from statement(stat)
raise NotImplementedError
# yield wasm.Statement('else')
# for stat in inp.else_statements:
# statement(wgn, stat)
yield wasm.Statement('end')
def statement(inp: ourlang.Statement) -> Statements:
def statement(wgn: WasmGenerator, inp: ourlang.Statement) -> None:
"""
Compile: any statement
"""
if isinstance(inp, ourlang.StatementReturn):
yield from statement_return(inp)
statement_return(wgn, inp)
return
if isinstance(inp, ourlang.StatementIf):
yield from statement_if(inp)
statement_if(wgn, inp)
return
if isinstance(inp, ourlang.StatementPass):
@ -329,27 +341,15 @@ def function(inp: ourlang.Function) -> wasm.Function:
"""
assert not inp.imported
wgn = WasmGenerator()
if isinstance(inp, ourlang.TupleConstructor):
statements = [
*_generate_tuple_constructor(inp)
]
locals_ = [
('___new_reference___addr', wasm.WasmTypeInt32(), ),
]
_generate_tuple_constructor(wgn, inp)
elif isinstance(inp, ourlang.StructConstructor):
statements = [
*_generate_struct_constructor(inp)
]
locals_ = [
('___new_reference___addr', wasm.WasmTypeInt32(), ),
]
_generate_struct_constructor(wgn, inp)
else:
statements = [
x
for y in inp.statements
for x in statement(y)
]
locals_ = [] # FIXME: Implement function locals, if required
for stat in inp.statements:
statement(wgn, stat)
return wasm.Function(
inp.name,
@ -358,9 +358,12 @@ def function(inp: ourlang.Function) -> wasm.Function:
function_argument(x)
for x in inp.posonlyargs
],
locals_,
[
(k, v.wasm_type(), )
for k, v in wgn.locals.items()
],
type_(inp.returns),
statements
wgn.statements
)
def module(inp: ourlang.Module) -> wasm.Module:
@ -389,12 +392,15 @@ def module(inp: ourlang.Module) -> wasm.Module:
return result
def _generate_tuple_constructor(inp: ourlang.TupleConstructor) -> Statements:
yield wasm.Statement('i32.const', str(inp.tuple.alloc_size()))
yield wasm.Statement('call', '$stdlib.alloc.__alloc__')
def _generate_tuple_constructor(wgn: WasmGenerator, inp: ourlang.TupleConstructor) -> None:
tmp_var = wgn.temp_var_i32('tuple_adr')
yield wasm.Statement('local.set', '$___new_reference___addr')
# Allocated the required amounts of bytes in memory
wgn.i32.const(inp.tuple.alloc_size())
wgn.call(stdlib_alloc.__alloc__)
wgn.local.set(tmp_var)
# Store each member individually
for member in inp.tuple.members:
mtyp = LOAD_STORE_TYPE_MAP.get(member.type.__class__)
if mtyp is None:
@ -402,18 +408,22 @@ def _generate_tuple_constructor(inp: ourlang.TupleConstructor) -> Statements:
# as members of struct or tuples
raise NotImplementedError(expression, inp, member)
yield wasm.Statement('local.get', '$___new_reference___addr')
yield wasm.Statement('local.get', f'$arg{member.idx}')
yield wasm.Statement(f'{mtyp}.store', 'offset=' + str(member.offset))
wgn.local.get(tmp_var)
wgn.add_statement('local.get', f'$arg{member.idx}')
wgn.add_statement(f'{mtyp}.store', 'offset=' + str(member.offset))
yield wasm.Statement('local.get', '$___new_reference___addr')
# Return the allocated address
wgn.local.get(tmp_var)
def _generate_struct_constructor(inp: ourlang.StructConstructor) -> Statements:
yield wasm.Statement('i32.const', str(inp.struct.alloc_size()))
yield wasm.Statement('call', '$stdlib.alloc.__alloc__')
def _generate_struct_constructor(wgn: WasmGenerator, inp: ourlang.StructConstructor) -> None:
tmp_var = wgn.temp_var_i32('tuple_adr')
yield wasm.Statement('local.set', '$___new_reference___addr')
# Allocated the required amounts of bytes in memory
wgn.i32.const(inp.struct.alloc_size())
wgn.call(stdlib_alloc.__alloc__)
wgn.local.set(tmp_var)
# Store each member individually
for member in inp.struct.members:
mtyp = LOAD_STORE_TYPE_MAP.get(member.type.__class__)
if mtyp is None:
@ -421,8 +431,9 @@ def _generate_struct_constructor(inp: ourlang.StructConstructor) -> Statements:
# as members of struct or tuples
raise NotImplementedError(expression, inp, member)
yield wasm.Statement('local.get', '$___new_reference___addr')
yield wasm.Statement('local.get', f'${member.name}')
yield wasm.Statement(f'{mtyp}.store', 'offset=' + str(member.offset))
wgn.local.get(tmp_var)
wgn.add_statement('local.get', f'${member.name}')
wgn.add_statement(f'{mtyp}.store', 'offset=' + str(member.offset))
yield wasm.Statement('local.get', '$___new_reference___addr')
# Return the allocated address
wgn.local.get(tmp_var)

View File

@ -3,6 +3,8 @@ Contains the syntax tree for ourlang
"""
from typing import Dict, List, Tuple
import enum
from typing_extensions import Final
WEBASSEMBLY_BUILDIN_FLOAT_OPS: Final = ('abs', 'sqrt', 'ceil', 'floor', 'trunc', 'nearest', )
@ -225,6 +227,37 @@ class AccessTupleMember(Expression):
self.varref = varref
self.member = member
class Fold(Expression):
"""
A (left or right) fold
"""
class Direction(enum.Enum):
"""
Which direction to fold in
"""
LEFT = 0
RIGHT = 1
dir: Direction
func: 'Function'
base: Expression
iter: Expression
def __init__(
self,
type_: TypeBase,
dir_: Direction,
func: 'Function',
base: Expression,
iter_: Expression,
) -> None:
super().__init__(type_)
self.dir = dir_
self.func = func
self.base = base
self.iter = iter_
class Statement:
"""
A statement within a function

View File

@ -38,6 +38,8 @@ from .ourlang import (
StructConstructor, TupleConstructor,
UnaryOp, VariableReference,
Fold,
Statement,
StatementIf, StatementPass, StatementReturn,
)
@ -348,7 +350,7 @@ class OurVisitor:
raise NotImplementedError(f'{node} as expr in FunctionDef')
def visit_Module_FunctionDef_Call(self, module: Module, function: Function, our_locals: OurLocals, exp_type: TypeBase, node: ast.Call) -> Union[FunctionCall, UnaryOp]:
def visit_Module_FunctionDef_Call(self, module: Module, function: Function, our_locals: OurLocals, exp_type: TypeBase, node: ast.Call) -> Union[Fold, FunctionCall, UnaryOp]:
if node.keywords:
_raise_static_error(node, 'Keyword calling not supported') # Yet?
@ -386,6 +388,37 @@ class OurVisitor:
'len',
self.visit_Module_FunctionDef_expr(module, function, our_locals, module.types['bytes'], node.args[0]),
)
elif node.func.id == 'foldl':
# TODO: This should a much more generic function!
# For development purposes, we're assuming you're doing a foldl(Callable[[u8, u8], u8], u8, bytes)
# In the future, we should probably infer the type of the second argument,
# and use it as expected types for the other u8s and the Iterable[u8] (i.e. bytes)
if not isinstance(exp_type, TypeUInt8):
_raise_static_error(node, f'Cannot make {node.func.id} result in {exp_type} - not implemented yet')
if 3 != len(node.args):
_raise_static_error(node, f'Function {node.func.id} requires 3 arguments but {len(node.args)} are given')
t_u8 = module.types['u8']
# TODO: This is not generic
subnode = node.args[0]
if not isinstance(subnode, ast.Name):
raise NotImplementedError(f'Calling methods that are not a name {subnode}')
if not isinstance(subnode.ctx, ast.Load):
_raise_static_error(subnode, 'Must be load context')
if subnode.id not in module.functions:
_raise_static_error(subnode, 'Reference to undefined function')
func = module.functions[subnode.id]
return Fold(
exp_type,
Fold.Direction.LEFT,
func,
self.visit_Module_FunctionDef_expr(module, function, our_locals, t_u8, node.args[1]),
self.visit_Module_FunctionDef_expr(module, function, our_locals, module.types['bytes'], node.args[2]),
)
else:
if node.func.id not in module.functions:
_raise_static_error(node, 'Call to undefined function')

View File

@ -127,7 +127,7 @@ class TypeTuple(TypeBase):
"""
Generates an internal name for this tuple
"""
mems = '@'.join('?' for x in self.members)
mems = '@'.join('?' for x in self.members) # FIXME: Should not be a questionmark
assert ' ' not in mems, 'Not implement yet: subtuples'
return f'tuple@{mems}'

View File

@ -16,29 +16,70 @@ class VarType_Base:
self.name = name
self.name_ref = f'${name}'
class VarType_u8(VarType_Base):
wasm_type = wasm.WasmTypeInt32
class VarType_i32(VarType_Base):
wasm_type = wasm.WasmTypeInt32
class Generator_i32:
def __init__(self, generator: 'Generator') -> None:
class Generator_i32i64:
def __init__(self, prefix: str, generator: 'Generator') -> None:
self.prefix = prefix
self.generator = generator
# 2.4.1. Numeric Instructions
# ibinop
self.add = functools.partial(self.generator.add, 'i32.add')
self.add = functools.partial(self.generator.add_statement, f'{prefix}.add')
# irelop
self.eq = functools.partial(self.generator.add, 'i32.eq')
self.ne = functools.partial(self.generator.add, 'i32.ne')
self.lt_u = functools.partial(self.generator.add, 'i32.lt_u')
self.eq = functools.partial(self.generator.add_statement, f'{prefix}.eq')
self.ne = functools.partial(self.generator.add_statement, f'{prefix}.ne')
self.lt_u = functools.partial(self.generator.add_statement, f'{prefix}.lt_u')
# 2.4.4. Memory Instructions
self.load = functools.partial(self.generator.add, 'i32.load')
self.load8_u = functools.partial(self.generator.add, 'i32.load8_u')
self.store = functools.partial(self.generator.add, 'i32.store')
self.load = functools.partial(self.generator.add_statement, f'{prefix}.load')
self.load8_u = functools.partial(self.generator.add_statement, f'{prefix}.load8_u')
self.store = functools.partial(self.generator.add_statement, f'{prefix}.store')
def const(self, value: int, comment: Optional[str] = None) -> None:
self.generator.add('i32.const', f'0x{value:08x}', comment=comment)
self.generator.add_statement(f'{self.prefix}.const', f'0x{value:08x}', comment=comment)
class Generator_i32(Generator_i32i64):
def __init__(self, generator: 'Generator') -> None:
super().__init__('i32', generator)
class Generator_i64(Generator_i32i64):
def __init__(self, generator: 'Generator') -> None:
super().__init__('i64', generator)
class Generator_f32f64:
def __init__(self, prefix: str, generator: 'Generator') -> None:
self.prefix = prefix
self.generator = generator
# 2.4.1. Numeric Instructions
# fbinop
self.add = functools.partial(self.generator.add_statement, f'{prefix}.add')
# frelop
self.eq = functools.partial(self.generator.add_statement, f'{prefix}.eq')
self.ne = functools.partial(self.generator.add_statement, f'{prefix}.ne')
# 2.4.4. Memory Instructions
self.load = functools.partial(self.generator.add_statement, f'{prefix}.load')
self.store = functools.partial(self.generator.add_statement, f'{prefix}.store')
def const(self, value: float, comment: Optional[str] = None) -> None:
# FIXME: Is this sufficient to guarantee the float comes across properly?
self.generator.add_statement(f'{self.prefix}.const', f'{value}', comment=comment)
class Generator_f32(Generator_f32f64):
def __init__(self, generator: 'Generator') -> None:
super().__init__('f32', generator)
class Generator_f64(Generator_f32f64):
def __init__(self, generator: 'Generator') -> None:
super().__init__('f64', generator)
class Generator_Local:
def __init__(self, generator: 'Generator') -> None:
@ -46,17 +87,17 @@ class Generator_Local:
# 2.4.3. Variable Instructions
def get(self, variable: VarType_Base, comment: Optional[str] = None) -> None:
self.generator.add('local.get', variable.name_ref, comment=comment)
self.generator.add_statement('local.get', variable.name_ref, comment=comment)
def set(self, variable: VarType_Base, comment: Optional[str] = None) -> None:
self.generator.locals.setdefault(variable.name, variable)
self.generator.add('local.set', variable.name_ref, comment=comment)
self.generator.add_statement('local.set', variable.name_ref, comment=comment)
def tee(self, variable: VarType_Base, comment: Optional[str] = None) -> None:
self.generator.locals.setdefault(variable.name, variable)
self.generator.add('local.tee', variable.name_ref, comment=comment)
self.generator.add_statement('local.tee', variable.name_ref, comment=comment)
class GeneratorBlock:
def __init__(self, generator: 'Generator', name: str) -> None:
@ -64,11 +105,11 @@ class GeneratorBlock:
self.name = name
def __enter__(self) -> None:
self.generator.add(self.name)
self.generator.add_statement(self.name)
def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
if not exc_type:
self.generator.add('end')
self.generator.add_statement('end')
class Generator:
def __init__(self) -> None:
@ -76,29 +117,46 @@ class Generator:
self.locals: Dict[str, VarType_Base] = {}
self.i32 = Generator_i32(self)
self.i64 = Generator_i64(self)
self.f32 = Generator_f32(self)
self.f64 = Generator_f64(self)
# 2.4.3 Variable Instructions
self.local = Generator_Local(self)
# 2.4.5 Control Instructions
self.nop = functools.partial(self.add, 'nop')
self.unreachable = functools.partial(self.add, 'unreachable')
self.nop = functools.partial(self.add_statement, 'nop')
self.unreachable = functools.partial(self.add_statement, 'unreachable')
# block
# loop
self.if_ = functools.partial(GeneratorBlock, self, 'if')
# br
# br_if
# br_table
self.return_ = functools.partial(self.add, 'return')
self.return_ = functools.partial(self.add_statement, 'return')
# call
# call_indirect
def call(self, function: wasm.Function) -> None:
self.add('call', f'${function.name}')
self.add_statement('call', f'${function.name}')
def add(self, name: str, *args: str, comment: Optional[str] = None) -> None:
def add_statement(self, name: str, *args: str, comment: Optional[str] = None) -> None:
self.statements.append(wasm.Statement(name, *args, comment=comment))
def temp_var_i32(self, infix: str) -> VarType_i32:
idx = 0
while (varname := f'__{infix}_tmp_var_{idx}__') in self.locals:
idx += 1
return VarType_i32(varname)
def temp_var_u8(self, infix: str) -> VarType_u8:
idx = 0
while (varname := f'__{infix}_tmp_var_{idx}__') in self.locals:
idx += 1
return VarType_u8(varname)
def func_wrapper(exported: bool = True) -> Callable[[Any], wasm.Function]:
"""
This wrapper will execute the function and return

View File

@ -411,7 +411,9 @@ def testEntry(f: bytes) -> bytes:
result = Suite(code_py).run_code(b'This is a test')
assert 4 == result.returned_value
# THIS DEPENDS ON THE ALLOCATOR
# A different allocator will return a different value
assert 20 == result.returned_value
@pytest.mark.integration_test
def test_bytes_length():