lib/mbedtls
1
0
Fork 0
mirror of https://github.com/Mbed-TLS/mbedtls.git synced 2025-08-08 17:42:09 +03:00
Code Activity

analyze_outcomes: reset name of TestLog to Results

Browse Source 
Signed-off-by: Valerio Setti <valerio.setti@nordicsemi.no>
This commit is contained in:
Valerio Setti
2023-10-17 11:03:16 +02:00
parent 3f33989762
commit f075e47bc1
1 changed files with 33 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
tests/scripts/analyze_outcomes.py
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Analyze the test outcomes from a full CI run. """Analyze the test outcomes from a full CI run.
@@ -15,7 +15,7 @@ import os
import check_test_cases import check_test_cases
class TestLog: class Results:
"""Process analysis results.""" """Process analysis results."""
def __init__(self): def __init__(self):
@@ -56,27 +56,27 @@ class TestCaseOutcomes:
""" """
return len(self.successes) + len(self.failures) return len(self.successes) + len(self.failures)
def execute_reference_driver_tests(log: TestLog, ref_component, driver_component, \ def execute_reference_driver_tests(results: Results, ref_component, driver_component, \
outcome_file) -> TestLog: outcome_file) -> Results:
"""Run the tests specified in ref_component and driver_component. Results """Run the tests specified in ref_component and driver_component. Results
are stored in the output_file and they will be used for the following are stored in the output_file and they will be used for the following
coverage analysis""" coverage analysis"""
# If the outcome file already exists, we assume that the user wants to # If the outcome file already exists, we assume that the user wants to
# perform the comparison analysis again without repeating the tests. # perform the comparison analysis again without repeating the tests.
if os.path.exists(outcome_file): if os.path.exists(outcome_file):
log.info("Outcome file (" + outcome_file + ") already exists. " + \ results.info("Outcome file (" + outcome_file + ") already exists. " + \
"Tests will be skipped.") "Tests will be skipped.")
return log return results
shell_command = "tests/scripts/all.sh --outcome-file " + outcome_file + \ shell_command = "tests/scripts/all.sh --outcome-file " + outcome_file + \
" " + ref_component + " " + driver_component " " + ref_component + " " + driver_component
log.info("Running: " + shell_command) results.info("Running: " + shell_command)
ret_val = subprocess.run(shell_command.split(), check=False).returncode ret_val = subprocess.run(shell_command.split(), check=False).returncode
if ret_val != 0: if ret_val != 0:
log.error("failed to run reference/driver components") results.error("failed to run reference/driver components")
return log return results
def analyze_coverage(results, outcomes, allow_list, full_coverage): def analyze_coverage(results, outcomes, allow_list, full_coverage):
"""Check that all available test cases are executed at least once.""" """Check that all available test cases are executed at least once."""
@@ -95,7 +95,7 @@ def analyze_coverage(results, outcomes, allow_list, full_coverage):
else: else:
results.warning('Allow listed test case was executed: {}', key) results.warning('Allow listed test case was executed: {}', key)
def analyze_driver_vs_reference(log: TestLog, outcomes, def analyze_driver_vs_reference(results: Results, outcomes,
component_ref, component_driver, component_ref, component_driver,
ignored_suites, ignored_test=None): ignored_suites, ignored_test=None):
"""Check that all tests executed in the reference component are also """Check that all tests executed in the reference component are also
@@ -130,15 +130,15 @@ def analyze_driver_vs_reference(log: TestLog, outcomes,
if component_ref in entry: if component_ref in entry:
reference_test_passed = True reference_test_passed = True
if(reference_test_passed and not driver_test_passed): if(reference_test_passed and not driver_test_passed):
log.error(key) results.error(key)
return log return results
def analyze_outcomes(log: TestLog, outcomes, args) -> TestLog: def analyze_outcomes(results: Results, outcomes, args) -> Results:
"""Run all analyses on the given outcome collection.""" """Run all analyses on the given outcome collection."""
analyze_coverage(log, outcomes, args['allow_list'], analyze_coverage(results, outcomes, args['allow_list'],
args['full_coverage']) args['full_coverage'])
return log return results
def read_outcome_file(outcome_file): def read_outcome_file(outcome_file):
"""Parse an outcome file and return an outcome collection. """Parse an outcome file and return an outcome collection.
@@ -161,30 +161,30 @@ by a semicolon.
outcomes[key].failures.append(setup) outcomes[key].failures.append(setup)
return outcomes return outcomes
def do_analyze_coverage(log: TestLog, outcome_file, args) -> TestLog: def do_analyze_coverage(results: Results, outcome_file, args) -> Results:
"""Perform coverage analysis.""" """Perform coverage analysis."""
log.info("\n*** Analyze coverage ***\n") results.info("\n*** Analyze coverage ***\n")
outcomes = read_outcome_file(outcome_file) outcomes = read_outcome_file(outcome_file)
log = analyze_outcomes(log, outcomes, args) results = analyze_outcomes(results, outcomes, args)
return log return results
def do_analyze_driver_vs_reference(log: TestLog, outcome_file, args) -> TestLog: def do_analyze_driver_vs_reference(results: Results, outcome_file, args) -> Results:
"""Perform driver vs reference analyze.""" """Perform driver vs reference analyze."""
log.info("\n*** Analyze driver {} vs reference {} ***\n".format( results.info("\n*** Analyze driver {} vs reference {} ***\n".format(
args['component_driver'], args['component_ref'])) args['component_driver'], args['component_ref']))
log = execute_reference_driver_tests(log, args['component_ref'], \ results = execute_reference_driver_tests(results, args['component_ref'], \
args['component_driver'], outcome_file) args['component_driver'], outcome_file)
ignored_suites = ['test_suite_' + x for x in args['ignored_suites']] ignored_suites = ['test_suite_' + x for x in args['ignored_suites']]
outcomes = read_outcome_file(outcome_file) outcomes = read_outcome_file(outcome_file)
log = analyze_driver_vs_reference(log, outcomes, results = analyze_driver_vs_reference(results, outcomes,
args['component_ref'], args['component_driver'], args['component_ref'], args['component_driver'],
ignored_suites, args['ignored_tests']) ignored_suites, args['ignored_tests'])
return log return results
# List of tasks with a function that can handle this task and additional arguments if required # List of tasks with a function that can handle this task and additional arguments if required
KNOWN_TASKS = { KNOWN_TASKS = {
@@ -649,7 +649,7 @@ KNOWN_TASKS = {
} }
def main(): def main():
main_log = TestLog() main_results = Results()
try: try:
parser = argparse.ArgumentParser(description=__doc__) parser = argparse.ArgumentParser(description=__doc__)
@@ -688,14 +688,13 @@ def main():
for task in tasks_list: for task in tasks_list:
test_function = KNOWN_TASKS[task]['test_function'] test_function = KNOWN_TASKS[task]['test_function']
test_args = KNOWN_TASKS[task]['args'] test_args = KNOWN_TASKS[task]['args']
main_log = test_function(main_log, options.outcomes, test_args) main_results = test_function(main_results, options.outcomes, test_args)
main_log.info("Overall results:\n" + \ main_results.info("Overall results:\n" + \
"{} warnings\n".format(main_log.warning_count) + \ "{} warnings\n".format(main_results.warning_count) + \
"{} errors\n".format(main_log.error_count)) "{} errors\n".format(main_results.error_count))
main_log.print_output() sys.exit(0 if (main_results.error_count == 0) else 2)
sys.exit(0 if (main_log.error_count == 0) else 2)
except Exception: # pylint: disable=broad-except except Exception: # pylint: disable=broad-except
# Print the backtrace and exit explicitly with our chosen status. # Print the backtrace and exit explicitly with our chosen status.