From 2482702d15f60982a44f3cea664db78bd4d90c67 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Tue, 25 Sep 2018 18:49:23 +0200 Subject: [PATCH] Test program for psa_constant_names Test psa_constant_names on many inputs. For each input, find out the numerical value by compiling and running a C program, pass the numerical value to psa_constant_names and compare the output with the original input. Gather inputs by parsing psa/crypto.h and test_suite_psa_crypto_metadata.data. For macros that take an argument, list some possible arguments using the parsed data. --- tests/scripts/test_psa_constant_names.py | 241 +++++++++++++++++++++++ 1 file changed, 241 insertions(+) create mode 100755 tests/scripts/test_psa_constant_names.py diff --git a/tests/scripts/test_psa_constant_names.py b/tests/scripts/test_psa_constant_names.py new file mode 100755 index 0000000000..59292428fa --- /dev/null +++ b/tests/scripts/test_psa_constant_names.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +'''Test the program psa_constant_names. +Gather constant names from header files and test cases. Compile a C program +to print out their numerical values, feed these numerical values to +psa_constant_names, and check that the output is the original name. +Return 0 if all test cases pass, 1 if the output was not always as expected, +or 1 (with a Python backtrace) if there was an operational error.''' + +import argparse +import itertools +import os +import platform +import re +import subprocess +import sys +import tempfile + +class Inputs: + '''Accumulate information about macros to test. +This includes macro names as well as information about their arguments +when applicable.''' + def __init__(self): + # Sets of names per type + self.statuses = set(['PSA_SUCCESS']) + self.algorithms = set(['0xffffffff']) + self.ecc_curves = set(['0xffff']) + self.key_types = set(['0xffffffff']) + self.key_usage_flags = set(['0x80000000']) + # Hard-coded value for an unknown hash algorithm + self.hash_algorithms = set(['0x010000ff']) + # Identifier prefixes + self.table_by_prefix = { + 'ERROR': self.statuses, + 'ALG': self.algorithms, + 'CURVE': self.ecc_curves, + 'KEY_TYPE': self.key_types, + 'KEY_USAGE': self.key_usage_flags, + } + # macro name -> list of argument names + self.argspecs = {} + # argument name -> list of values + self.arguments_for = {} + + def gather_arguments(self): + '''Populate the list of values for macro arguments. +Call this after parsing all the inputs.''' + self.arguments_for['hash_alg'] = sorted(self.hash_algorithms) + self.arguments_for['curve'] = sorted(self.ecc_curves) + + def format_arguments(self, name, arguments): + '''Format a macro call with arguments..''' + return name + '(' + ', '.join(arguments) + ')' + + def distribute_arguments(self, name): + '''Generate macro calls with each tested argument set. +If name is a macro without arguments, just yield "name". +If name is a macro with arguments, yield a series of "name(arg1,...,argN)" +where each argument takes each possible value at least once.''' + if name not in self.argspecs: + yield name + return + argspec = self.argspecs[name] + if argspec == []: + yield name + '()' + return + argument_lists = [self.arguments_for[arg] for arg in argspec] + arguments = [values[0] for values in argument_lists] + yield self.format_arguments(name, arguments) + for i in range(len(arguments)): + for value in argument_lists[i][1:]: + arguments[i] = value + yield self.format_arguments(name, arguments) + arguments[i] = argument_lists[0] + + # Regex for interesting header lines. + # Groups: 1=macro name, 2=type, 3=argument list (optional). + header_line_re = \ + re.compile(r'#define +' + + r'(PSA_((?:KEY_)?[A-Z]+)_\w+)' + + r'(?:\(([^\n()]*)\))?') + # Regex of macro names to exclude. + excluded_name_re = re.compile('_(?:GET|IS|OF)_|_(?:BASE|FLAG|MASK)\Z') + argument_split_re = re.compile(r' *, *') + def parse_header_line(self, line): + '''Parse a C header line, looking for "#define PSA_xxx".''' + m = re.match(self.header_line_re, line) + if not m: + return + name = m.group(1) + if re.search(self.excluded_name_re, name): + return + dest = self.table_by_prefix.get(m.group(2)) + if dest is None: + return + dest.add(name) + if m.group(3): + self.argspecs[name] = re.split(self.argument_split_re, m.group(3)) + + def parse_header(self, filename): + '''Parse a C header file, looking for "#define PSA_xxx".''' + with open(filename, 'r') as input: + for line in input: + self.parse_header_line(line) + + def add_test_case_line(self, function, argument): + '''Parse a test case data line, looking for algorithm metadata tests.''' + if function.endswith('_algorithm'): + self.algorithms.add(argument) + if function == 'hash_algorithm': + self.hash_algorithms.add(argument) + elif function == 'key_type': + self.key_types.add(argument) + elif function == 'ecc_key_types': + self.ecc_curves.add(argument) + + # Regex matching a *.data line containing a test function call and + # its arguments. The actual definition is partly positional, but this + # regex is good enough in practice. + test_case_line_re = re.compile('(?!depends_on:)(\w+):([^\n :][^:\n]*)') + def parse_test_cases(self, filename): + '''Parse a test case file (*.data), looking for algorithm metadata tests.''' + with open(filename, 'r') as input: + for line in input: + m = re.match(self.test_case_line_re, line) + if m: + self.add_test_case_line(m.group(1), m.group(2)) + +def gather_inputs(headers, test_suites): + '''Read the list of inputs to test psa_constant_names with.''' + inputs = Inputs() + for header in headers: + inputs.parse_header(header) + for test_cases in test_suites: + inputs.parse_test_cases(test_cases) + inputs.gather_arguments() + return inputs + +def remove_file_if_exists(filename): + '''Remove the specified file, ignoring errors.''' + if not filename: + return + try: + os.remove(filename) + except: + pass + +def run_c(options, names): + '''Generate and run a program to print out numerical values for names.''' + c_name = None + exe_name = None + try: + c_fd, c_name = tempfile.mkstemp(suffix='.c', + dir='programs/psa') + exe_suffix = '.exe' if platform.system() == 'Windows' else '' + exe_name = c_name[:-2] + exe_suffix + remove_file_if_exists(exe_name) + c_file = os.fdopen(c_fd, 'w', encoding='ascii') + c_file.write('''/* Generated by test_psa_constant_names.py */ +#include +#include +int main(void) +{ +''') + for name in names: + c_file.write(' printf("0x%08x\\n", {});\n'.format(name)) + c_file.write(''' return 0; +} +''') + c_file.close() + cc = os.getenv('CC', 'cc') + subprocess.check_call([cc] + + ['-I' + dir for dir in options.include] + + ['-o', exe_name, c_name]) + os.remove(c_name) + output = subprocess.check_output([exe_name]) + return output.decode('ascii').strip().split('\n') + finally: + remove_file_if_exists(exe_name) + +normalize_strip_re = re.compile(r'\s+') +def normalize(expr): + '''Normalize the C expression so as not to care about trivial differences. +Currently "trivial differences" means whitespace.''' + expr = re.sub(normalize_strip_re, '', expr, len(expr)) + return expr.strip().split('\n') + +def do_test(options, inputs, type, names): + '''Test psa_constant_names for the specified type. +Run program on names. +Use inputs to figure out what arguments to pass to macros that take arguments.''' + names = sorted(itertools.chain(*map(inputs.distribute_arguments, names))) + values = run_c(options, names) + output = subprocess.check_output([options.program, type] + values) + outputs = output.decode('ascii').strip().split('\n') + errors = [(type, name, value, output) + for (name, value, output) in zip(names, values, outputs) + if normalize(name) != normalize(output)] + return len(names), errors + +def report_errors(errors): + '''Describe each case where the output is not as expected.''' + for type, name, value, output in errors: + print('For {} "{}", got "{}" (value: {})' + .format(type, name, output, value)) + +def run_tests(options, inputs): + '''Run psa_constant_names on all the gathered inputs. +Return a tuple (count, errors) where count is the total number of inputs +that were tested and errors is the list of cases where the output was +not as expected.''' + count = 0 + errors = [] + for type, names in [('status', inputs.statuses), + ('algorithm', inputs.algorithms), + ('ecc_curve', inputs.ecc_curves), + ('key_type', inputs.key_types), + ('key_usage', inputs.key_usage_flags)]: + c, e = do_test(options, inputs, type, names) + count += c + errors += e + return count, errors + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=globals()['__doc__']) + parser.add_argument('--include', '-I', + action='append', default=['include'], + help='Directory for header files') + parser.add_argument('--program', + default='programs/psa/psa_constant_names', + help='Program to test') + options = parser.parse_args() + headers = [os.path.join(options.include[0], 'psa/crypto.h')] + test_suites = ['tests/suites/test_suite_psa_crypto_metadata.data'] + inputs = gather_inputs(headers, test_suites) + count, errors = run_tests(options, inputs) + report_errors(errors) + if errors == []: + print('{} test cases PASS'.format(count)) + else: + print('{} test cases, {} FAIL'.format(count, len(errors))) + exit(1)