Johan B.W. de Vries 97b61e3ee1 Test generation framework with typing improvements
Prior to this PR, each type would have its own handwritten
test suite. The end result was that not all types were tested
for all situations.

This PR adds a framework based on a Markdown file, which
generates the basic tests for the types defined in json
files. These are auto generated and updated by the Makefile
before the test suite is run.

Also, a number of unsupported type combinations are now
supported.

Also, we now support negative literals.

Also, allocation calculation fixes for nested types.

Also, the test helpers can now properly import and export
typed variables such as bytes, static arrays and tuples. This
may come in handy when it comes to phasm platform wanting to
route data.

Also, adds better support for i8 type.

Also, started on a runtime.py, since there's quite some code
now that deals with compile time handling of WebAssembly stuff.

Also, minor improvement to the type constrains, namely we
better match 'tuple' literals with static array types.

Also, reduced spam when printing the type analysis results;
constraints that go back on the backlog are now no longer
printed one by one. It now also prints the end results of
the typing analysis.

Also, reorganized the big test_primitives test into type
classes.

Also, replaced pylint with ruff.
2023-11-15 12:52:23 +01:00

155 lines
4.5 KiB
Python

import functools
import json
import sys
from typing import Any
import marko
import marko.md_renderer
def get_tests(template):
test_data = None
for el in template.children:
if isinstance(el, marko.block.BlankLine):
continue
if isinstance(el, marko.block.Heading):
if test_data is not None:
yield test_data
test_data = []
test_data.append(el)
continue
if test_data is not None:
test_data.append(el)
if test_data is not None:
yield test_data
def apply_settings(settings, txt):
for k, v in settings.items():
if k in ('CODE_HEADER', 'PYTHON'):
continue
txt = txt.replace(f'${k}', v)
return txt
def generate_assertion_expect(result, arg, given=None):
given = given or []
result.append('result = Suite(code_py).run_code(' + ', '.join(repr(x) for x in given) + ')')
result.append(f'assert {repr(arg)} == result.returned_value')
def generate_assertion_expect_type_error(result, error_msg, error_comment = None):
result.append('with pytest.raises(Type3Exception) as exc_info:')
result.append(' Suite(code_py).run_code()')
result.append(f'assert {repr(error_msg)} == exc_info.value.args[0][0].msg')
result.append(f'assert {repr(error_comment)} == exc_info.value.args[0][0].comment')
def json_does_not_support_byte_or_tuple_values_fix(inp: Any):
if isinstance(inp, (int, float, )):
return inp
if isinstance(inp, str):
if inp.startswith('bytes:'):
return inp[6:].encode()
return inp
if isinstance(inp, list):
return tuple(map(json_does_not_support_byte_or_tuple_values_fix, inp))
if isinstance(inp, dict):
return {
key: json_does_not_support_byte_or_tuple_values_fix(val)
for key, val in inp.items()
}
raise NotImplementedError(inp)
def generate_assertions(settings, result_code):
result = []
locals_ = {
'TYPE': settings['TYPE'],
'TYPE_NAME': settings['TYPE_NAME'],
'expect': functools.partial(generate_assertion_expect, result),
'expect_type_error': functools.partial(generate_assertion_expect_type_error, result),
}
if 'PYTHON' in settings:
locals_.update(json_does_not_support_byte_or_tuple_values_fix(settings['PYTHON']))
if 'VAL0' not in locals_:
locals_['VAL0'] = eval(settings['VAL0'])
exec(result_code, {}, locals_)
return ' ' + '\n '.join(result) + '\n'
def generate_code(markdown, template, settings):
type_name = settings['TYPE_NAME']
print('"""')
print('AUTO GENERATED')
print()
print('TEMPLATE:', sys.argv[1])
print('SETTINGS:', sys.argv[2])
print('"""')
print('import pytest')
print()
print('from phasm.type3.entry import Type3Exception')
print()
print('from ..helpers import Suite')
print()
for test in get_tests(template):
assert len(test) == 4, test
heading, paragraph, code_block1, code_block2 = test
assert isinstance(heading, marko.block.Heading)
assert isinstance(paragraph, marko.block.Paragraph)
assert isinstance(code_block1, marko.block.FencedCode)
assert isinstance(code_block2, marko.block.FencedCode)
test_id = apply_settings(settings, heading.children[0].children)
user_story = apply_settings(settings, markdown.renderer.render(paragraph))
inp_code = apply_settings(settings, code_block1.children[0].children)
result_code = markdown.renderer.render_children(code_block2)
print('@pytest.mark.integration_test')
print(f'def test_{type_name}_{test_id}():')
print(' """')
print(' ' + user_story.replace('\n', '\n '))
print(' """')
print(' code_py = """')
if 'CODE_HEADER' in settings:
for lin in settings['CODE_HEADER']:
print(lin)
print()
print(inp_code.rstrip('\n'))
print('"""')
print()
print(generate_assertions(settings, result_code))
print()
def main():
markdown = marko.Markdown(
renderer=marko.md_renderer.MarkdownRenderer,
)
with open(sys.argv[1], 'r', encoding='utf-8') as fil:
template = markdown.parse(fil.read())
with open(sys.argv[2], 'r', encoding='utf-8') as fil:
settings = json.load(fil)
if 'TYPE_NAME' not in settings:
settings['TYPE_NAME'] = settings['TYPE']
generate_code(markdown, template, settings)
if __name__ == '__main__':
main()