diff --git a/.github/workflows/test-arch.yml b/.github/workflows/test-arch.yml index 99aab93a5..00a830790 100644 --- a/.github/workflows/test-arch.yml +++ b/.github/workflows/test-arch.yml @@ -59,7 +59,7 @@ jobs: - name: Install run: | sudo apt-get update - sudo DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y perl sudo libxml-checker-perl libyaml-perl rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config make gcc ccache meson git liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev + sudo DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y perl sudo libxml-checker-perl libyaml-perl python3-yaml rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config make gcc ccache meson git liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev docker run --privileged --rm tonistiigi/binfmt --install all - name: Login to GHCR diff --git a/.github/workflows/test-bsd.yml b/.github/workflows/test-bsd.yml index e3408accf..999333acf 100644 --- a/.github/workflows/test-bsd.yml +++ b/.github/workflows/test-bsd.yml @@ -42,6 +42,10 @@ jobs: brew install -q pkg-config openssl@3 libpq libxml2 libyaml meson + # PyYAML for the test harness. Homebrew has no formula for it and its python is externally managed, so this is the + # supported way to add a package to it. + pip3 install --break-system-packages pyyaml + # The PostgreSQL versions supported by pgBackRest, oldest first, read from the generated header so the list never needs to # be updated here pgSupport=$(sed -n 's/^#define PG_VERSION_[0-9_]*_Z *"\([0-9.]*\)".*/postgresql@\1/p' \ @@ -138,6 +142,19 @@ jobs: sudo pkg install -y bash git ${pgServer%-server}-client ${pgServer} pkgconf libxml2 gmake perl5 libyaml \ p5-YAML-LibYAML rsync meson + # PyYAML for the test harness. The package is py-pyyaml, renamed from py-yaml in 2024 so both names are + # searched, and it is built only for the python flavors the ports tree selects, which is not necessarily the version the + # python3 meta package provides. Ask which one is available rather than assuming, preferring the newest. + pyYaml=$({ pkg rquery -x '%n' '^py[0-9]+-pyyaml$'; pkg rquery -x '%n' '^py[0-9]+-yaml$'; } | sort -V | tail -1) + pyBin=/usr/local/bin/python$(echo ${pyYaml} | sed 's/-.*$//;s/^py\([0-9]\)/\1./') + echo "pyyaml package: ${pyYaml}, interpreter: ${pyBin}" + sudo pkg install -y ${pyYaml} + + # The ports that depend on python install only a versioned interpreter so nothing provides python3, which is how the test + # harness is invoked. Link it to the interpreter PyYAML was installed for. + sudo ln -sf ${pyBin} /usr/local/bin/python3 + python3 -c 'import sys, yaml; print("python %s, PyYAML %s" % (sys.version.split()[0], yaml.__version__))' + - name: Run Test shell: cpa.sh {0} run: cd .. && perl ${GITHUB_WORKSPACE?}/test/test.pl --vm-max=2 --no-coverage --no-valgrind --module=command --test=backup --test=info --test=archive-push diff --git a/.github/workflows/test-misc.yml b/.github/workflows/test-misc.yml index 203b471bd..c508ef909 100644 --- a/.github/workflows/test-misc.yml +++ b/.github/workflows/test-misc.yml @@ -43,9 +43,9 @@ jobs: with: path: pgbackrest - # Check that the smoke test, the only Python file, is formatted. On failure the exact command to fix it is printed so it can - # be copied and run locally. This runs before the C format check because that check runs ci.pl, which removes passwordless - # sudo, and installing black needs it. + # Check that the Python is formatted. On failure the exact command to fix it is printed so it can be copied and run locally. + # This runs before the C format check because that check runs ci.pl, which removes passwordless sudo, and installing black + # needs it. - name: Check Python Format run: | sudo apt-get update @@ -53,11 +53,11 @@ jobs: cd ${GITHUB_WORKSPACE?}/pgbackrest - if ! black --check --diff --line-length=132 test/smoke.py; then + if ! black --check --diff --line-length=132 test/smoke.py test/test.py test/lib test/src/module/test; then echo - echo "ERROR: test/smoke.py is not formatted correctly. Run this from the pgbackrest path to fix it:" + echo "ERROR: Python is not formatted correctly. Run this from the pgbackrest path to fix it:" echo - echo " black --line-length=132 test/smoke.py" + echo " black --line-length=132 test/smoke.py test/test.py test/lib test/src/module/test" echo exit 1 fi diff --git a/.gitignore b/.gitignore index 4dab0bf7b..7adfb1484 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *~ *.swp .DS_Store +__pycache__/ diff --git a/doc/xml/contributing.xml b/doc/xml/contributing.xml index 00161bec2..34fb844b9 100644 --- a/doc/xml/contributing.xml +++ b/doc/xml/contributing.xml @@ -95,6 +95,7 @@ autoconf-archive libssl-dev zlib1g-dev libxml2-dev libpq-dev pkg-config libxml-checker-perl libyaml-perl libdbd-pg-perl liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev libyaml-dev ccache meson + python3-yaml python3-coverage -y 2>&1 diff --git a/meson.build b/meson.build index d9006de04..449a049ff 100644 --- a/meson.build +++ b/meson.build @@ -287,10 +287,9 @@ subdir('src') # out all subdirs, which excludes the smoke test from them. subdir('test') -# The doc and test tools generate code and are not built from a distribution tarball +# The doc tool generates code and is not built from a distribution tarball if not dist subdir('doc/src') - subdir('test/src') # Bake the generated files and the dist marker into the tarball so it builds without the code generation tooling meson.add_dist_script('src/build/dist.sh') diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 000000000..ef77dce00 --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,6 @@ +{ + "include": ["test"], + "extraPaths": ["test/lib"], + "pythonVersion": "3.9", + "typeCheckingMode": "off" +} diff --git a/test/ci.pl b/test/ci.pl index b428cbe56..90e407f2f 100755 --- a/test/ci.pl +++ b/test/ci.pl @@ -133,7 +133,8 @@ eval processBegin('install common packages'); processExec('sudo apt-get -qq update', {bSuppressStdErr => true, bSuppressError => true}); processExec( - 'sudo DEBIAN_FRONTEND=noninteractive apt-get install -y meson libxml-checker-perl libyaml-perl', {bSuppressStdErr => true}); + 'sudo DEBIAN_FRONTEND=noninteractive apt-get install -y meson libxml-checker-perl libyaml-perl python3-yaml', + {bSuppressStdErr => true}); processEnd(); if (!$bNoTempFs) diff --git a/test/define.yaml b/test/define.yaml index 8e021f4ce..008e83ec7 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -724,27 +724,6 @@ unit: - build/postgres/parse - build/postgres/render - # ******************************************************************************************************************************** - - name: test - - test: - # ---------------------------------------------------------------------------------------------------------------------------- - - name: test - total: 3 - - coverage: - - test/command/test/build - - test/command/test/define - - test/command/test/lint - - test/command/test/test - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: coverage - total: 1 - - coverage: - - test/command/test/coverage - # ******************************************************************************************************************************** - name: info @@ -1063,6 +1042,125 @@ unit: - doc/command/build/reference - doc/command/build/varStore + # ******************************************************************************************************************************** + # Tests for the test harness itself. These are python so they are run directly rather than built, which means no total is needed + # since there are no runs to select. The harness is the same everywhere so there is nothing to gain from running these on every vm. + # + # This group is last so that nothing it declares can affect the modules compiled into any other test. + # ******************************************************************************************************************************** + - name: test + lang: python + vm: [none, u24, rh9] + + test: + # ---------------------------------------------------------------------------------------------------------------------------- + - name: common/error + + coverage: + - test/common/error + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: common/exec + + coverage: + - test/common/exec + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: common/log + + coverage: + - test/common/log + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: common/storage + + coverage: + - test/common/storage + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: common/string + + coverage: + - test/common/string_id + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: common/render + + coverage: + - test/common/render + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: config/cli + + coverage: + - test/config/cli + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: config/config + + coverage: + - test/config/config + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: command/test/define + + coverage: + - test/command/test/define + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: command/lint/lint + + coverage: + - test/command/lint/ascii + - test/command/lint/string_id + - test/command/lint/lint + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: command/test/python + + coverage: + - test/command/test/python + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: command/coverage/report + + coverage: + - test/command/coverage/report + + # The report renders the coverage model, which is defined with the command that builds it + depend: + - test/command/coverage/coverage + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: command/coverage/filter_py + + coverage: + - test/command/coverage/filter_py + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: command/coverage/filter_c + + coverage: + - test/command/coverage/filter_c + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: command/coverage/coverage + + coverage: + - test/command/coverage/coverage + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: command/test/build + + coverage: + - test/command/test/build + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: command/test/unit + + coverage: + - test/command/test/unit + # ********************************************************************************************************************************** # Integration tests # diff --git a/test/lib/command/coverage/coverage.py b/test/lib/command/coverage/coverage.py new file mode 100644 index 000000000..43a16cd09 --- /dev/null +++ b/test/lib/command/coverage/coverage.py @@ -0,0 +1,344 @@ +"""Coverage Testing and Reporting. + +Each unit test writes gcov JSON into test/result/coverage/raw as it completes; this merges them, applies the coverage exceptions, +warns about anything not fully covered, and writes the report.""" + +#################################################################################################################################### +import json +import os + +from command.test.define import TEST_LANG_C, TEST_LANG_PYTHON, test_def_find, test_def_parse +from common.error import check +from common.log import WARN, log +from common.storage import file_read, file_write, path_list + + +#################################################################################################################################### +class CoverageLine: + """A line of code, with a hit count for the line and for each branch on it. + + A python branch is an arc to another line, so target_list names where each branch goes and the report can say which path was + never taken. C branches have no such target and the list is None.""" + + def __init__(self, no, hit, branch_list, target_list=None): + self.no = no + self.hit = hit + self.branch_list = branch_list # Hit count per branch, or None when the line has no branches + self.target_list = target_list # Line each branch goes to, for python, or None for C + + +#################################################################################################################################### +class CoverageFunction: + """A function, with the range of lines it spans and how many times it was called.""" + + def __init__(self, name, line_begin, line_end, hit): + self.name = name + self.line_begin = line_begin + self.line_end = line_end + self.hit = hit + + +#################################################################################################################################### +class CoverageFile: + """A code module, with the lines and functions coverage was collected for and the totals over them.""" + + def __init__(self, name, lang=TEST_LANG_C): + self.name = name + self.lang = lang + self.line_list = [] + self.function_list = [] + + self.function_hit = 0 + self.function_total = 0 + self.branch_hit = 0 + self.branch_total = 0 + self.line_hit = 0 + self.line_total = 0 + + ################################################################################################################################ + def covered(self): + """Is every line and branch in this file covered?""" + + return self.line_hit == self.line_total and self.branch_hit == self.branch_total + + +#################################################################################################################################### +def _file_merge_python(file, merge_file): + """Merge python coverage for a file, keyed by line number. + + Which lines a run reports depends on how the module was loaded, e.g. a module that ran as the program reports fewer statements + than the same module imported by a test, so a run may contribute a line the others never reported. Functions are not collected + for python so there is nothing to merge for them.""" + + line_map = {line.no: line for line in file.line_list} + + for merge_line in merge_file.line_list: + line = line_map.get(merge_line.no) + + if line is None: + file.line_list.append(merge_line) + line_map[merge_line.no] = merge_line + + continue + + line.hit += merge_line.hit + + _branch_merge_python(line, merge_line) + + file.line_list.sort(key=lambda line: line.no) + + +#################################################################################################################################### +def _branch_merge_python(line, merge_line): + """Merge the python branches of a line, which are arcs to another line, keyed by where the arc goes. + + An arc that exists only when an exception is raised is unknown to a run that did not raise one, so an arc reported by one run + and not another is added rather than being an error.""" + + if merge_line.branch_list is None: + return + + if line.branch_list is None: + line.branch_list = list(merge_line.branch_list) + line.target_list = list(merge_line.target_list) + + return + + branch = dict(zip(line.target_list, line.branch_list)) + + for target, hit in zip(merge_line.target_list, merge_line.branch_list): + branch[target] = branch.get(target, 0) + hit + + line.target_list = sorted(branch) + line.branch_list = [branch[target] for target in line.target_list] + + +#################################################################################################################################### +class Coverage: + """Coverage for every code module reported, merged from the runs that produced it.""" + + def __init__(self): + self.file_list = [] + + ################################################################################################################################ + def file_find(self, name): + """Find a file by code module name, or None when it is not present.""" + + for file in self.file_list: + if file.name == name: + return file + + return None + + ################################################################################################################################ + def merge(self, merge): + """Merge coverage from another run. + + A file already present is summed line by line and branch by branch, which requires the two runs to have been built from the + same source.""" + + for merge_file in merge.file_list: + file = self.file_find(merge_file.name) + + # If the file does not exist yet then take it as is + if file is None: + self.file_list.append(merge_file) + + continue + + # Python reports what it knows about rather than what the module contains, so two runs of the same file may not report + # the same lines and are merged by line number + if file.lang == TEST_LANG_PYTHON: + _file_merge_python(file, merge_file) + + continue + + check( + len(file.line_list) == len(merge_file.line_list) and len(file.function_list) == len(merge_file.function_list), + "coverage for '%s' does not match the prior run" % file.name, + ) + + for line, merge_line in zip(file.line_list, merge_file.line_list): + check(line.no == merge_line.no, "coverage line mismatch in '%s'" % file.name) + + line.hit += merge_line.hit + + if line.branch_list is not None: + check( + merge_line.branch_list is not None and len(line.branch_list) == len(merge_line.branch_list), + "coverage branches for '%s' do not match the prior run" % file.name, + ) + + for branch_idx, merge_branch in enumerate(merge_line.branch_list): + line.branch_list[branch_idx] += merge_branch + + merge_function_list = {function.name: function for function in merge_file.function_list} + + for function in file.function_list: + merge_function = merge_function_list.get(function.name) + + check(merge_function is not None, "coverage for function '%s' is missing" % function.name) + check( + function.line_begin == merge_function.line_begin and function.line_end == merge_function.line_end, + "coverage for function '%s' does not match the prior run" % function.name, + ) + + function.hit += merge_function.hit + + self.file_list.sort(key=lambda file: file.name) + + ################################################################################################################################ + def total_calculate(self): + """Calculate the totals for every file.""" + + for file in self.file_list: + file.function_total = len(file.function_list) + file.function_hit = sum(1 for function in file.function_list if function.hit > 0) + file.line_total = len(file.line_list) + file.line_hit = sum(1 for line in file.line_list if line.hit > 0) + file.branch_total = 0 + file.branch_hit = 0 + + for line in file.line_list: + if line.branch_list is None: + continue + + file.branch_total += len(line.branch_list) + file.branch_hit += sum(1 for branch in line.branch_list if branch > 0) + + ################################################################################################################################ + def covered_remove(self): + """Remove lines that are fully covered so the report shows only what is missing.""" + + for file in self.file_list: + file.line_list = [ + line + for line in file.line_list + if line.hit == 0 or (line.branch_list is not None and any(branch == 0 for branch in line.branch_list)) + ] + + +#################################################################################################################################### +def coverage_module_file(name, lang=TEST_LANG_C): + """Map a code module name to the file name coverage reports it under. + + For example "common/error/error" becomes "src/common/error/error.c", and the python module "test/common/string_id" becomes + "test/lib/common/string_id.py".""" + + # The python the harness is written in lives in the library rather than beside the tests + if lang == TEST_LANG_PYTHON: + check(name.startswith("test/"), "python module '%s' must be under test/" % name) + + return "test/lib/" + name[len("test/") :] + ".py" + + if name.startswith("test/"): + result = "test/src/" + name[len("test/") :] + ".c" + elif name.startswith("doc/"): + result = "doc/src/" + name[len("doc/") :] + ".c" + else: + result = "src/" + name + ".c" + + # A vendored module is included rather than compiled, as is a module named .inc + if result.endswith(".vendor.c"): + result += ".inc" + elif result.endswith(".inc.c"): + result = result[: -len(".inc.c")] + ".c.inc" + + return result + + +#################################################################################################################################### +def _coverage_list_build(module_def_list, module_name_list): + """Build the file names of the code modules that should be fully covered by the tests that ran. + + A module is mapped to a file name here rather than by the caller since the mapping depends on the language of the test that + declared it.""" + + name_list = [] + result = [] + + for module_name in module_name_list: + module = test_def_find(module_def_list, module_name) + + for coverage in module.coverage_list: + if coverage.coverable and coverage.name not in name_list: + name_list.append(coverage.name) + result.append(coverage_module_file(coverage.name, module.lang)) + + # Remove code modules that require a test that was not run for full coverage + for module in module_def_list: + for coverage in module.coverage_list: + if coverage.name in name_list and module.name not in module_name_list: + log(WARN, "module '%s' did not have all tests run required for coverage" % coverage.name) + result.pop(name_list.index(coverage.name)) + name_list.remove(coverage.name) + + return result + + +#################################################################################################################################### +def cmd_coverage(config): + """Merge the coverage from the tests that ran and write the report. + + Returns 1 when a module is missing coverage, else 0.""" + + from command.coverage import filter_c, filter_py + from command.coverage.report import report_render, summary_render + + module_def_list = test_def_parse(config.repo_path) + coverage_module_list = _coverage_list_build(module_def_list, config.module) + + # Combine the coverage written by each test that ran. Both languages report into the same model so a run that covered C and + # python produces a single report. + path_raw = os.path.join(config.repo_path, "test/result/coverage/raw") + coverage = Coverage() + + for name in path_list(path_raw, expression=r"\.json$", error_on_missing=True): + raw = json.loads(file_read(os.path.join(path_raw, name))) + + # gcov reports files as an array while coverage.py reports them as an object keyed by path + filter = filter_c if isinstance(raw["files"], list) else filter_py + + coverage.merge(filter.coverage_read(raw, coverage_module_list)) + + # The source the tests were built from, which is the repository copy in the test path + def source_read(name): + return file_read(os.path.join(config.test_path, "repo", name)) + + # Each filter applies the exceptions for its own language and leaves the other language alone + filter_c.coverage_filter(coverage, source_read, config.coverage_summary, config.vm) + filter_py.coverage_filter(coverage, source_read, config.coverage_summary, config.vm) + + coverage.total_calculate() + + # Write coverage summary + if config.coverage_summary: + check(config.vm != "none", "coverage summary must be run in vm") + + file_write(os.path.join(config.repo_path, "doc/xml/auto/metric-coverage-report.auto.xml"), summary_render(coverage)) + + # Warn on missing coverage + result = 0 + + # A module that was expected to be covered but produced no coverage at all, e.g. because the test wrote no data, would + # otherwise pass silently since there is nothing to report as uncovered + for name in coverage_module_list: + if coverage.file_find(name) is None: + log(WARN, "module '%s' has no coverage data" % name) + result += 1 + + for file in coverage.file_list: + if not file.covered(): + log( + WARN, + "module '%s' is not fully covered (%u/%u lines, %u/%u branches)" + % (file.name, file.line_hit, file.line_total, file.branch_hit, file.branch_total), + ) + result += 1 + + # Write coverage report with the covered lines filtered out + coverage.covered_remove() + + file_write(os.path.join(config.repo_path, "test/result/coverage/coverage.html"), report_render(coverage, source_read)) + + return 1 if result > 0 else 0 diff --git a/test/lib/command/coverage/filter_c.py b/test/lib/command/coverage/filter_c.py new file mode 100644 index 000000000..6f1c9faa6 --- /dev/null +++ b/test/lib/command/coverage/filter_c.py @@ -0,0 +1,125 @@ +"""C Coverage Filter. + +Converts gcov JSON into the coverage model and applies the exceptions that make the report meaningful for C source: + +- Debug logging that was covered is dropped, since it is not interesting and would otherwise dominate the report. +- A line or branch marked uncovered/uncoverable in a comment is treated as covered. Only uncoverable is honored when building the + documentation summary, so the summary reports what genuinely cannot be covered. +- Branches in assertions and in macros that expand to more than they appear to are dropped, since they can never be taken both ways + in a test.""" + +#################################################################################################################################### +import re + +from command.coverage.coverage import Coverage, CoverageFile, CoverageFunction, CoverageLine +from command.test.define import TEST_LANG_C + +# Debug logging that is not interesting for coverage reporting +_LOG_EXP = re.compile(r"\s+FUNCTION_(LOG|TEST)_(VOID|BEGIN|END|PARAM(|_P|_PP))\(") +_LOG_RETURN_EXP = re.compile(r"\s+FUNCTION_(LOG|TEST)_RETURN_VOID\(") + + +#################################################################################################################################### +def _module_match(path, module): + """Does a path gcov reported come from a code module? + + A module is normally compiled from the repository so the path ends with the module file name. A shimmed .c.inc is compiled from + the copy the build writes to the unit include path, which has no src prefix so that it resolves ahead of the repository, and is + matched on the rest of the name.""" + + if path.endswith(module): + return True + + return module.startswith("src/") and module.endswith(".c.inc") and path.endswith(module[len("src/") :]) + + +#################################################################################################################################### +def coverage_read(raw, module_list): + """Read gcov JSON into a coverage object, keeping only the files that match a covered code module.""" + + result = Coverage() + + for file_raw in raw["files"]: + # gcov reports the path it compiled, which is longer than the module name the report uses + name = next((module for module in module_list if _module_match(file_raw["file"], module)), None) + + if name is None: + continue + + file = CoverageFile(name) + + for function_raw in file_raw["functions"]: + file.function_list.append( + CoverageFunction( + function_raw["name"], function_raw["start_line"], function_raw["end_line"], function_raw["execution_count"] + ) + ) + + for line_raw in file_raw["lines"]: + branch_list = None + + if line_raw.get("branches"): + branch_list = [branch["count"] for branch in line_raw["branches"]] + + file.line_list.append(CoverageLine(line_raw["line_number"], line_raw["count"], branch_list)) + + result.file_list.append(file) + + result.file_list.sort(key=lambda file: file.name) + + return result + + +#################################################################################################################################### +def _exception_exp(coverage_summary, vm): + """Build the expressions that match a coverage exception. + + Only uncoverable counts for the summary, and vm_covered only applies when not running in a vm, since there the code really is + covered.""" + + uncover_branch = "uncoverable_branch" if coverage_summary else "uncover(ed|able)_branch" + uncover = "uncoverable" if coverage_summary else "uncover(ed|able)" + vm_covered = "|vm_covered" if vm == "none" else "" + + branch = re.compile( + r"\s{4}[A-Z][A-Z0-9_]+\([^\?]*\)|\s{4}(ASSERT|CHECK|CHECK_FMT|assert|switch\s)\(|\{\+{0,1}(%s%s)" + % (uncover_branch, vm_covered) + ) + line = re.compile(r"\{\+{0,1}(%s%s)[^_]" % (uncover, vm_covered)) + + return branch, line + + +#################################################################################################################################### +def coverage_filter(coverage, source_read, coverage_summary, vm): + """Apply the coverage exceptions to every file, using the source the test was built from.""" + + branch_exp, line_exp = _exception_exp(coverage_summary, vm) + + for file in coverage.file_list: + if file.lang != TEST_LANG_C: + continue + + line_text_list = source_read(file.name).split("\n") + line_list = [] + + for line in file.line_list: + text = line_text_list[line.no - 1] + + # Remove covered lines for debug logging. These are not very interesting for coverage reporting. + if line.hit != 0: + if _LOG_EXP.search(text) or (_LOG_RETURN_EXP.search(text) and line_text_list[line.no] == "}"): + continue + + # If not covered then check for line coverage exceptions + if line.hit == 0 and line_exp.search(text): + line.hit = 1 + + # If not covered then check for branch coverage exceptions + if line.branch_list is not None and branch_exp.search(text): + # Remove branch coverage so it is not reported + line.branch_list = None + + line_list.append(line) + + file.line_list = line_list diff --git a/test/lib/command/coverage/filter_py.py b/test/lib/command/coverage/filter_py.py new file mode 100644 index 000000000..c2464d5dd --- /dev/null +++ b/test/lib/command/coverage/filter_py.py @@ -0,0 +1,106 @@ +"""Python Coverage Filter. + +Converts coverage.py JSON into the coverage model and applies the same uncovered/uncoverable annotations the C filter does, so both +languages are reported the same way and can be combined in one report. + +Two things differ from gcov and are worth knowing when reading a combined report: + +- Python records whether a line ran, not how many times, so a hit count here is only ever zero or one. The report never shows the + count so nothing is lost, but a merge cannot sum runs the way it does for C. +- Function coverage is not collected, since it is only reported in the documentation summary and that covers the project source + rather than the harness. +- A branch is an arc out of a line rather than a condition within an expression, so "if a and b" is one decision to python and two + branches to gcc. Full branch coverage therefore means less in python than it does in C. +""" + +#################################################################################################################################### +import re + +from command.test.define import TEST_LANG_PYTHON + + +#################################################################################################################################### +def coverage_read(raw, module_list): + """Read coverage.py JSON into a coverage object, keeping only the files that match a covered code module.""" + + from command.coverage.coverage import Coverage, CoverageFile, CoverageLine + + result = Coverage() + + for path, file_raw in raw["files"].items(): + # coverage.py reports the path it measured, which is longer than the module name the report uses + name = next((module for module in module_list if path.endswith(module)), None) + + if name is None: + continue + + file = CoverageFile(name, TEST_LANG_PYTHON) + + # Function coverage is not collected. It is only ever reported in the documentation summary, which covers the project + # source rather than the harness, so there is nothing to report it against. + + # Collect the branches on each line, which coverage.py reports as arcs from one line to another + branch = {} + + for source, target in file_raw["executed_branches"]: + branch.setdefault(source, []).append((target, 1)) + + for source, target in file_raw["missing_branches"]: + branch.setdefault(source, []).append((target, 0)) + + for no in sorted(set(file_raw["executed_lines"]) | set(file_raw["missing_lines"])): + hit = 1 if no in file_raw["executed_lines"] else 0 + branch_list = None + target_list = None + + if no in branch: + arc_list = sorted(branch[no]) + target_list = [target for target, _ in arc_list] + branch_list = [hit for _, hit in arc_list] + + file.line_list.append(CoverageLine(no, hit, branch_list, target_list)) + + result.file_list.append(file) + + result.file_list.sort(key=lambda file: file.name) + + return result + + +#################################################################################################################################### +def _exception_exp(coverage_summary): + """Build the expressions that match a coverage exception. + + Only uncoverable counts for the summary, so the summary reports what genuinely cannot be covered. There is no vm_covered here + since no python code is reachable only inside a container.""" + + uncover_branch = "uncoverable_branch" if coverage_summary else "uncover(ed|able)_branch" + uncover = "uncoverable" if coverage_summary else "uncover(ed|able)" + + return re.compile(r"\{\+{0,1}(%s)" % uncover_branch), re.compile(r"\{\+{0,1}(%s)[^_]" % uncover) + + +#################################################################################################################################### +def coverage_filter(coverage, source_read, coverage_summary, vm): + """Apply the coverage exceptions to every file, using the source the test was run from.""" + + branch_exp, line_exp = _exception_exp(coverage_summary) + + for file in coverage.file_list: + if file.lang != TEST_LANG_PYTHON: + continue + + line_text_list = source_read(file.name).split("\n") + + for line in file.line_list: + text = line_text_list[line.no - 1] + + # If not covered then check for line coverage exceptions + if line.hit == 0 and line_exp.search(text): + line.hit = 1 + + # If not covered then check for branch coverage exceptions + if line.branch_list is not None and branch_exp.search(text): + # Remove branch coverage so it is not reported + line.branch_list = None + line.target_list = None diff --git a/test/lib/command/coverage/report.py b/test/lib/command/coverage/report.py new file mode 100644 index 000000000..cc144a9d5 --- /dev/null +++ b/test/lib/command/coverage/report.py @@ -0,0 +1,416 @@ +"""Coverage Report. + +Renders the HTML report of everything that is not covered and the summary table used in the documentation.""" + +#################################################################################################################################### +import os + +TITLE = "pgBackRest Coverage Report" + +HTML_PRE = ( + '\n' + '\n' + "\n" + " " + TITLE + "\n" + ' \n' + ' \n" + "\n" + "\n" + '
' + TITLE + "
\n" +) + +TOC_PRE = ( + '
\n' + ' \n' + ' \n' + ' \n' + " \n" +) + +TOC_COVERED_PRE = ' \n \n \n" +TOC_UNCOVERED_PRE = ' \n \n \n" +TOC_POST = "
FILE
' +TOC_COVERED_POST = "
\n ' +TOC_UNCOVERED_POST = "\n
\n" + +RPT_PRE = '
' +RPT_MID2 = ( + "
\n" + ' \n' + ' \n' + ' \n' + ' \n' + ' \n' + " \n" +) + +RPT_LINE_PRE = ' \n \n \n' +RPT_BRANCH_UNCOVERED_PRE = '\n \n" +RPT_CODE = ' \n \n" +RPT_SKIP = ' \n \n \n' +RPT_POST = TOC_POST + +HTML_POST = "\n\n" + + +#################################################################################################################################### +def _context_render(no, text): + """Render a context line, i.e. a covered line shown around one that is missing coverage.""" + + return RPT_LINE_PRE + str(no) + RPT_BRANCH_COVERED + RPT_CODE + text + RPT_LINE_POST + + +#################################################################################################################################### +def _context_trim(text): + """Is this line one that may be trimmed from the context, i.e. it carries no information on its own?""" + + text = text.strip() + + return text == "" or text == "{" or text == "}" + + +#################################################################################################################################### +def _file_render(file, line_text_list): + """Render the report for one file.""" + + result = RPT_PRE + file.name + RPT_MID1 + file.name + RPT_MID2 + line_last = 0 + + for line_idx, line in enumerate(file.line_list): + # Check branch coverage and build report string + branch_covered = True + branch_str = "[" + + if line.branch_list is not None: + # A python branch is an arc to another line so name the line that was never jumped to, which says which path was + # missed. A C branch has no such target and is shown as one mark per outcome. + if line.target_list is not None: + for branch, target in zip(line.branch_list, line.target_list): + if branch != 0: + continue + + if len(branch_str) > 1: + branch_str += " " + + # A negative target is the end of the function the line is in, i.e. the branch that never returned. The value + # is the line the function is defined on, which is of no use in the report. + branch_str += "-> return" if target < 0 else "-> %d" % target + branch_covered = False + else: + for branch in line.branch_list: + if len(branch_str) > 1: + branch_str += " " + + if branch == 0: + branch_str += "-" + branch_covered = False + else: + branch_str += "+" + + branch_str += "]" + + # Add before context + context_begin = line.no - 5 if line.no > 5 else 1 + context_end = line.no - 1 if line.no > 1 else 0 + + if line_last != 0 and context_begin < line_last + 1: + context_begin = line_last + 1 + + if line_last != 0 and context_begin > line_last + 1: + result += RPT_SKIP + + # Trim uninteresting lines from the top of the context + for context_idx in range(context_begin, context_end + 1): + if not _context_trim(line_text_list[context_idx - 1]): + break + + context_begin = context_idx + 1 + + for context_idx in range(context_begin, context_end + 1): + result += _context_render(context_idx, line_text_list[context_idx - 1]) + + # Output coverage + result += RPT_LINE_PRE + str(line.no) + + if not branch_covered: + result += RPT_BRANCH_UNCOVERED_PRE + branch_str + RPT_BRANCH_UNCOVERED_POST + else: + result += RPT_BRANCH_COVERED + + result += RPT_CODE_UNCOVERED if line.hit == 0 else RPT_CODE + result += line_text_list[line.no - 1] + RPT_LINE_POST + + line_last = line.no + + # Add after context + context_begin = line.no + 1 + context_end = line.no + 5 if line.no <= len(line_text_list) - 5 else len(line_text_list) + + if line_idx < len(file.line_list) - 1: + line_next = file.line_list[line_idx + 1] + + if context_end >= line_next.no: + context_end = line_next.no - 1 + + # Trim uninteresting lines from the bottom of the context + for context_idx in range(context_end, context_begin - 1, -1): + if not _context_trim(line_text_list[context_idx - 1]): + break + + context_end = context_idx - 1 + + for context_idx in range(context_begin, context_end + 1): + result += _context_render(context_idx, line_text_list[context_idx - 1]) + + line_last = context_end + + return result + RPT_POST + + +#################################################################################################################################### +def report_render(coverage, source_read): + """Render the coverage report. + + Files that are fully covered are listed in the table of contents but have no report section.""" + + result = HTML_PRE + TOC_PRE + + # Build table of contents + for file in coverage.file_list: + if not file.line_list: + result += TOC_COVERED_PRE + file.name + TOC_COVERED_POST + else: + result += TOC_UNCOVERED_PRE + file.name + TOC_UNCOVERED_MID + file.name + TOC_UNCOVERED_POST + + result += TOC_POST + + # Build files that are missing coverage + for file in coverage.file_list: + if file.line_list: + result += _file_render(file, source_read(file.name).split("\n")) + + return result + HTML_POST + + +#################################################################################################################################### +# Coverage summary for the documentation +SUMMARY_ROW = ( + "\n" + " %s\n" + " %s\n" + " %s\n" + " %s\n" + "\n" +) + + +#################################################################################################################################### +def summary_value(hit, total): + """Render hit/total and the percentage, matching cvtPctToZ() in src/common/type/convert.c.""" + + if total == 0: + return "---" + + # A ratio of one is fixed at 100% so rounding cannot throw off the result + if hit == total: + percent = 10000 + else: + percent = (int(hit / total * 100000) + 5) // 10 + + percent = "%03u" % percent + + return "%u/%u (%s.%s%%)" % (hit, total, percent[:-2], percent[-2:]) + + +#################################################################################################################################### +def summary_render(coverage): + """Render the summary table, aggregating the files of each code module into a single row.""" + + result = "" + module_list = {} + total = [0, 0, 0, 0, 0, 0] + + for file in coverage.file_list: + # Filter out anything that is not project source + if file.name.startswith("test/") or file.name.startswith("doc/"): + continue + + name = os.path.dirname(file.name)[len("src/") :] + module = module_list.setdefault(name, [0, 0, 0, 0, 0, 0]) + value = (file.function_hit, file.function_total, file.branch_hit, file.branch_total, file.line_hit, file.line_total) + + for index, item in enumerate(value): + module[index] += item + total[index] += item + + for name in sorted(module_list): + module = module_list[name] + + result += ( + SUMMARY_ROW + % ( + name, + summary_value(module[0], module[1]), + summary_value(module[2], module[3]), + summary_value(module[4], module[5]), + ) + + "\n" + ) + + return result + SUMMARY_ROW % ( + "TOTAL", + summary_value(total[0], total[1]), + summary_value(total[2], total[3]), + summary_value(total[4], total[5]), + ) diff --git a/test/lib/command/lint/ascii.py b/test/lib/command/lint/ascii.py new file mode 100644 index 000000000..5b7f8b985 --- /dev/null +++ b/test/lib/command/lint/ascii.py @@ -0,0 +1,72 @@ +r"""ASCII Linter. + +Checks that source is 7-bit ASCII, which keeps out characters that could be used to hide code, e.g. invisible, zero-width, +bidirectional, or homoglyph characters. Intentional non-ASCII byte values must be written as \xNN escapes so they remain visible in +the source.""" + +#################################################################################################################################### +from common.log import WARN, log + +# Bytes that may appear in source: printable 7-bit ASCII plus tab and linefeed +_BYTE_ALLOW = bytes(range(0x20, 0x7F)) + b"\t\n" + + +#################################################################################################################################### +def lint_ascii(data): + """Check that a file is 7-bit ASCII and return the number of errors found. + + Tab and newline are the only control characters permitted.""" + + # Nothing to report unless a byte outside the allowed set is present. Checking first keeps the scan below off the hot path. + if not data.translate(None, delete=_BYTE_ALLOW): + return 0 + + result = 0 + line = 1 + index = 0 + size = len(data) + + while index < size: + char = data[index] + + # Count lines for reporting + if char == 0x0A: + line += 1 + index += 1 + + continue + + # Tab is the only other permitted control character + if char == 0x09 or 0x20 <= char <= 0x7E: + index += 1 + + continue + + # Decode the code point for reporting on a best-effort basis. Validation is unnecessary since the character is rejected + # regardless, so the byte count is taken from the lead byte and limited to the bytes available. + seq_size = 1 + + if char >= 0xF0: + seq_size = 4 + elif char >= 0xE0: + seq_size = 3 + elif char >= 0xC0: + seq_size = 2 + + if seq_size > size - index: + seq_size = size - index + + code_point = char + + if seq_size > 1: + code_point = char & (0xFF >> (seq_size + 1)) + + for seq_index in range(1, seq_size): + code_point = (code_point << 6) | (data[index + seq_index] & 0x3F) + + log(WARN, "line %u contains disallowed character U+%04X (source must be 7-bit ASCII)" % (line, code_point)) + result += 1 + + index += seq_size + + return result diff --git a/test/lib/command/lint/lint.py b/test/lib/command/lint/lint.py new file mode 100644 index 000000000..96e460b75 --- /dev/null +++ b/test/lib/command/lint/lint.py @@ -0,0 +1,65 @@ +"""Code Linter. + +Scans every file in the repository for content that could hide code from review and checks that StringId macros encode what they +claim to.""" + +#################################################################################################################################### +import os + +from command.lint.ascii import lint_ascii +from command.lint.string_id import lint_str_id +from common.error import TestError +from common.log import WARN, log +from common.storage import path_list_recurse + +# Files that are exempt from the content checks. A binary file is an unscannable place to hide content, so each entry must be a +# deliberate, reviewable decision. +_FILE_SKIP_LIST = ( + "doc/resource/logo.png", # Project logo + "test/data/filecopy.table.bin", # Binary test fixture + "doc/resource/git-history.cache", # Generated from git history, contains non-ASCII author names +) + + +#################################################################################################################################### +def _lint_str_id_apply(name): + """Should the StringId check be applied to this file? + + It applies to C source, including hand-written .c.inc, but not to generated or vendored includes.""" + + if name.endswith(".c") or name.endswith(".h"): + return True + + return name.endswith(".c.inc") and not name.endswith(".auto.c.inc") and not name.endswith(".vendor.c.inc") + + +#################################################################################################################################### +def cmd_lint(path_repo): + """Lint every file in the repository.""" + + for name in path_list_recurse(path_repo): + # Skip files that are exempt from the content checks + if name in _FILE_SKIP_LIST: + continue + + # Read the file, skipping any that cannot be read. Committed files are always readable after checkout, so this only occurs + # from local filesystem state, and readability is enforced by the build rather than the linter. + try: + with open(os.path.join(path_repo, name), "rb") as file: + data = file.read() + except OSError: + continue + + # A binary file cannot be scanned for hidden content, so it is not allowed unless added to the skip list + if b"\x00" in data: + log(WARN, "unexpected binary file (add to the linter skip list if intentional)") + error_total = 1 + # Otherwise the file must be 7-bit ASCII text, with the StringId check applied to C source + else: + error_total = lint_ascii(data) + + if _lint_str_id_apply(name): + error_total += lint_str_id(data.decode("utf-8", errors="replace")) + + if error_total > 0: + raise TestError("%u linter error(s) in '%s' (see warnings above)" % (error_total, name)) diff --git a/test/lib/command/lint/string_id.py b/test/lib/command/lint/string_id.py new file mode 100644 index 000000000..b2a2673cd --- /dev/null +++ b/test/lib/command/lint/string_id.py @@ -0,0 +1,71 @@ +"""StringId Linter. + +Checks that every StringId macro in the source encodes the string it claims to. The encoded value is written into the source so it +can be used in a switch, which means a hand-edited string and value can drift apart without the compiler noticing.""" + +#################################################################################################################################### +import re + +from common.error import TestError +from common.log import WARN, log +from common.render import bld_str_id_seq + +# Macro invocations to check, e.g. STRID5("test", 0x2a7250) or STRID6S("test", 1, 0x1e2a7250) +_STRID_EXP = re.compile(r"STRID(5|6)(S?)\([^)]+\)") + +# The macro definitions themselves, which name their parameters rather than encoding a string +_STRID_DEFINE_LIST = ( + "STRID5(str, strId)", + "STRID5S(str, seq, strId)", + "STRID6(str, strId)", + "STRID6S(str, seq, strId)", +) + + +#################################################################################################################################### +def lint_str_id(source): + """Check the StringId macros in a source file and return the number of errors found.""" + + result = 0 + + for match in _STRID_EXP.finditer(source): + text = match.group(0) + param_list = [param.strip() for param in text[text.find("(") + 1 : -1].split(",")] + param = param_list[0] + + # Skip macro definitions + if text in _STRID_DEFINE_LIST: + continue + + # Skip test strings, i.e. a quoted string inside a string + if param.startswith('\\"') and param.endswith('\\"'): + continue + + # Skip test values + if len(param_list) > 1 and param_list[1].startswith("TEST_"): + continue + + # Param must begin and end with a quote + if not param.startswith('"') or not param.endswith('"'): + log(WARN, "'%s' must have quotes around string parameter '%s'" % (text, param)) + result += 1 + + continue + + # Check validity of the string + try: + if match.group(2) == "": + expected = bld_str_id_seq(param[1:-1]) + else: + expected = bld_str_id_seq(param[1:-1], int(param_list[1])) + except TestError as error: + log(WARN, "'%s' is not valid: %s" % (text, error)) + result += 1 + + continue + + if text != expected: + log(WARN, "'%s' should be '%s'" % (text, expected)) + result += 1 + + return result diff --git a/test/lib/command/test/build.py b/test/lib/command/test/build.py new file mode 100644 index 000000000..7837c6f49 --- /dev/null +++ b/test/lib/command/test/build.py @@ -0,0 +1,514 @@ +"""Test Build Handler. + +Generates everything the unit build needs into the unit path: the shimmed copies of the modules the harness needs to reach into, the +harnesses that include them, meson.build, and test.c. + +Files are only written when the content changes so an unchanged file keeps its timestamp and ninja does not rebuild the world. +Anything left in the unit path that this test did not generate is removed, since the path is reused across tests.""" + +#################################################################################################################################### +import os + +from command.test.define import TEST_TYPE_INTEGRATION, TEST_TYPE_PERFORMANCE, TEST_TYPE_UNIT +from common.error import check +from common.log import log_level_enum +from common.render import bld_enum +from common.storage import file_read, file_remove, file_write_differs, path_list, path_list_recurse + +# Comment block used to divide the generated meson.build, matching the width used in the rest of the project +MESON_COMMENT_BLOCK = "#" * 132 + + +#################################################################################################################################### +def path_module(module_name): + """Map a code module name to its path in the repository, e.g. "common/error/error" becomes "src/common/error/error".""" + + if module_name.startswith("test/"): + return "test/src" + module_name[len("test") :] + + if module_name.startswith("doc/"): + return "doc/src" + module_name[len("doc") :] + + return "src/" + module_name + + +#################################################################################################################################### +def path_relative(base, compare): + """Generate a relative path from the base path to the compare path.""" + + check(base != compare, "base and compare paths may not be equal") + + base_list = base.split("/") + compare_list = compare.split("/") + index = 0 + + # Find the part of the paths that is the same + while index < len(base_list) and index < len(compare_list) and base_list[index] == compare_list[index]: + index += 1 + + return "/".join([".."] * (len(base_list) - index) + compare_list[index:]) + + +#################################################################################################################################### +def build_shim(shim_c, function_list): + """Rewrite the shimmed functions in a module so the harness can wrap them. + + The function definition is renamed with a _SHIMMED suffix and a forward declaration is added ahead of it. A static function + keeps a declaration under its original name so the module can still call it, while an extern function is declared under the + shimmed name so there is no missing-prototypes warning.""" + + result = "" + in_list = shim_c.split("\n") + + check(in_list[-1] == "", "shim module must end with a linefeed") + + for in_idx, line in enumerate(in_list): + found = False + + for function in function_list: + if not line.startswith(function) or line.find("(") != len(function): + continue + + check(in_idx > 0, "shimmed function may not be on the first line") + found = True + + # Add the forward declaration, which runs from the function name to the line before the opening brace + result += " " + + if in_list[in_idx - 1].startswith("static "): + result += line + else: + result += function + "_SHIMMED" + line[len(function) :] + + scan_idx = in_idx + 1 + + while True: + # In a properly formatted C file the end of the list can never be reached + check(scan_idx < len(in_list), "unable to find the end of the function signature") + + if in_list[scan_idx] == "{": + break + + if in_list[scan_idx - 1].endswith(","): + result += " " + + result += in_list[scan_idx].strip() + scan_idx += 1 + + result += "; " + in_list[in_idx - 1] + + # Alter the function name so it can be shimmed + result += "\n" + function + "_SHIMMED" + line[len(function) :] + break + + # Just copy the line when a function is not found + if not found: + if in_idx != 0: + result += "\n" + + result += line + + return result + + +#################################################################################################################################### +class TestBuild: + """Build the unit test into the unit path.""" + + def __init__(self, config, module, architecture): + self.path_repo = config.repo_path + self.path_test = config.test_path + self.vm = "none" if module.type == TEST_TYPE_INTEGRATION else config.vm + self.vm_int = config.vm # Vm used for integration and the pgbackrest binary + self.vm_id = config.vm_id + self.pg_version = config.pg_version + self.module = module + self.test = config.test + self.scale = config.scale + self.log_level = config.log_level_test + self.log_time = config.log_timestamp + self.time_zone = config.tz + self.architecture = architecture + self.coverage = config.coverage + self.profile = config.profile + self.optimize = config.optimize + self.back_trace = config.back_trace + + self.path_unit = os.path.join(self.path_test, "unit-%u" % self.vm_id, self.vm) + self.path_unit_build = os.path.join(self.path_unit, "build") + + # Files generated into the unit path, so anything else there can be removed + self._file_list = [] + + ################################################################################################################################ + def _write(self, file, content): + """Write a file into the unit path and record that it belongs to this test.""" + + file_write_differs(os.path.join(self.path_unit, file), content) + self._file_list.append(file) + + ################################################################################################################################ + def _read_repo(self, file): + """Read a file from the repository.""" + + return file_read(os.path.join(self.path_repo, file)) + + ################################################################################################################################ + def _build_shim(self): + """Write the shimmed copy of every module a harness shims.""" + + for shim in self.module.shim_list: + # Skip this shim for integration tests + if self.module.type == TEST_TYPE_INTEGRATION and not shim.integration: + continue + + # The shim module .c is always shimmed and written since the harness includes it. A function may instead be defined in + # an included .c.inc module, in which case that module is shimmed as well. + shim_module_list = [shim.name] + + for function in shim.function_list: + if function.inc is not None: + module = os.path.dirname(shim.name) + "/" + function.inc + + if module not in shim_module_list: + shim_module_list.append(module) + + for shim_module_idx, shim_module in enumerate(shim_module_list): + # The first module is the shim module .c; the rest are included .c.inc modules + included = shim_module_idx != 0 + + # Get the functions to shim in this module + function_list = [] + + for function in shim.function_list: + function_module = shim.name if function.inc is None else os.path.dirname(shim.name) + "/" + function.inc + + if function_module == shim_module: + function_list.append(function.name) + + # Read the source. An included module is a .c.inc, otherwise a .c. + read_file = path_module(shim_module) + ".c" + (".inc" if included else "") + + # Write the shimmed source. The shim module .c keeps its normal path. An included .c.inc uses the include path so it + # resolves to this copy (via the unit include dir, before the repo copy) when the shim module includes it. + write_file = shim_module + ".c.inc" if included else read_file + + self._write(write_file, build_shim(self._read_repo(read_file), function_list)) + + ################################################################################################################################ + def _build_harness(self): + """Write the harnesses that include shimmed modules. + + Returns the harnesses that apply to this test paired with the source to compile for each, and every module they include. A + harness with includes is copied into the unit path with the includes filled in, otherwise it is compiled from the + repository.""" + + harness_list = [] + harness_include_list = [] + path_repo_rel = path_relative(self.path_unit, self.path_repo) + + for harness in self.module.harness_list: + # Skip this harness for integration tests + if self.module.type == TEST_TYPE_INTEGRATION and not harness.integration: + continue + + harness_file = "test/src/harness/%s.c" % harness.name + + # If there are includes then copy and update the harness + if harness.include_list: + include_replace = [] + + for include in harness.include_list: + # A shimmed module is included from the unit path, everything else straight from the repository + path = self.path_unit if self.module.shim_find(include) is not None else self.path_repo + include_replace.append('#include "%s/%s.c"' % (path, path_module(include))) + harness_include_list.append(include) + + harness_c = self._read_repo(harness_file).replace("{[SHIM_MODULE]}", "\n".join(include_replace)) + + self._write(harness_file, harness_c) + harness_list.append((harness, harness_file)) + # Else harness can be referenced directly from the repo path + else: + harness_list.append((harness, "%s/%s" % (path_repo_rel, harness_file))) + + return harness_list, harness_include_list + + ################################################################################################################################ + def _build_meson(self, harness_list, harness_include_list): + """Write meson.build for the unit.""" + + module = self.module + path_repo_rel = path_relative(self.path_unit, self.path_repo) + + self._write("meson_options.txt", self._read_repo("meson_options.txt")) + + meson_build = self._read_repo("meson.build") + + # Comment out subdirs that are not used for testing + meson_build = meson_build.replace("subdir('", "# subdir('") + + # Comment out the distribution script, which is only used when building a distribution tarball + meson_build = meson_build.replace("meson.add_dist_script(", "# meson.add_dist_script(") + + if not self.back_trace: + meson_build = meson_build.replace( + " configuration.set('HAVE_LIBBACKTRACE'", "# configuration.set('HAVE_LIBBACKTRACE'" + ) + + # Write build.auto.in + meson_build += ( + "\n" + + MESON_COMMENT_BLOCK + + "\n# Write configuration\n" + + MESON_COMMENT_BLOCK + + "\n" + + "configure_file(output: 'build.auto.h', configuration: configuration)\n" + + "\n" + + "add_global_arguments('-DFN_EXTERN=extern', language : 'c')\n" + + "add_global_arguments('-DVR_EXTERN_DECLARE=extern', language : 'c')\n" + + "add_global_arguments('-DVR_EXTERN_DEFINE=', language : 'c')\n" + + "add_global_arguments('-DERROR_MESSAGE_BUFFER_SIZE=131072', language : 'c')\n" + ) + + # Configure features + if module.feature is not None: + meson_build += "add_global_arguments('-DHRN_INTEST_%s', language : 'c')\n" % module.feature + + for feature in module.feature_list: + meson_build += "add_global_arguments('-DHRN_FEATURE_%s', language : 'c')\n" % feature + + # Add compiler flags + if module.flag is not None: + meson_build += "add_global_arguments('%s', language : 'c')\n" % module.flag + + # Add coverage + if self.coverage: + meson_build += "add_global_arguments('-DDEBUG_COVERAGE', language : 'c')\n" + + # Add container flag + if self.vm != "none": + meson_build += "add_global_arguments('-DTEST_CONTAINER_REQUIRED', language : 'c')\n" + + # Build unit test + meson_build += "\n" + MESON_COMMENT_BLOCK + "\n# Unit test\n" + MESON_COMMENT_BLOCK + "\n" + "src_unit = files(\n" + + for depend in module.depend_list: + if depend in harness_include_list: + continue + + meson_build += " '%s/%s.c',\n" % (path_repo_rel, path_module(depend)) + + # Add harnesses + for harness, harness_file in harness_list: + # Add harness depends + harness_depend_path = "test/src/harness/%s" % harness.name + + for name in path_list(os.path.join(self.path_repo, harness_depend_path), expression=r"\.c$"): + meson_build += " '%s/%s/%s',\n" % (path_repo_rel, harness_depend_path, name) + + # Skip the harness when one of its includes is covered or included by the module, since test.c includes it directly + if any(module.coverage_find(include) is not None or include in module.include_list for include in harness.include_list): + continue + + meson_build += " '%s',\n" % harness_file + + meson_build += ( + " '%s/test/src/harness/test.c',\n" % path_repo_rel + + " 'test.c',\n" + + ")\n" + + "\n" + + "executable(\n" + + " 'test-unit',\n" + + " sources: src_unit,\n" + ) + + # Add C args + c_arg = "" + + if self.optimize or module.type == TEST_TYPE_PERFORMANCE: + c_arg += "\n '-O2'," + + if self.profile: + c_arg += "\n '-pg',\n '-no-pie'," + + if c_arg: + meson_build += " c_args: [%s\n ],\n" % c_arg + + # Add linker args + link_arg = "" + + if self.profile: + link_arg += "\n '-pg',\n '-no-pie'," + + if link_arg: + meson_build += " link_args: [%s\n ],\n" % link_arg + + meson_build += ( + " include_directories:\n" + + " include_directories(\n" + + " '.',\n" + + " '%s/src',\n" % path_repo_rel + + " '%s/doc/src',\n" % path_repo_rel + + " '%s/test/src',\n" % path_repo_rel + + " ),\n" + + " dependencies: [\n" + ) + + if self.back_trace: + meson_build += " lib_backtrace,\n" + + meson_build += ( + " lib_bz2,\n" + + " lib_openssl,\n" + + " lib_lz4,\n" + + " lib_pq,\n" + + " lib_ssh2,\n" + + " lib_xml,\n" + + " lib_yaml,\n" + + " lib_z,\n" + + " lib_zstd,\n" + + " ],\n" + + ")\n" + ) + + self._write("meson.build", meson_build) + + ################################################################################################################################ + def _build_test_c(self, harness_list): + """Write test.c, the wrapper that includes the code under test and the test module itself.""" + + module = self.module + path_repo_rel = path_relative(self.path_unit, self.path_repo) + test_c = self._read_repo("test/src/test.c") + + # Enable debug test trace + if not self.profile and module.type != TEST_TYPE_PERFORMANCE: + test_c = test_c.replace("{[C_TEST_DEBUG_TEST_TRACE]}", "#define DEBUG_TEST_TRACE") + else: + test_c = test_c.replace("{[C_TEST_DEBUG_TEST_TRACE]}", "// Debug test trace not enabled") + + # Files to test/include. A file covered by a harness is pulled in by including that harness instead. + test_include_file_list = [ + coverage.name for coverage in module.coverage_list if coverage.coverable and not coverage.included + ] + list(module.include_list) + + test_include_file = [] + harness_included = [] + + for include in test_include_file_list: + # A module pulled in by a harness is covered by including that harness rather than the module itself + harness_file = next((file for harness, file in harness_list if include in harness.include_list), None) + + if harness_file is not None: + if harness_file not in harness_included: + test_include_file.append('#include "%s"' % harness_file) + harness_included.append(harness_file) + else: + test_include_file.append('#include "%s/%s.c"' % (path_repo_rel, path_module(include))) + + replace = { + "{[C_INCLUDE]}": "\n".join(test_include_file), + # Test path + "{[C_TEST_PATH]}": os.path.join(self.path_test, "test-%u" % self.vm_id), + # Harness data path + "{[C_HRN_PATH]}": os.path.join(self.path_test, "data-%u" % self.vm_id), + # Harness repo path + "{[C_HRN_PATH_REPO]}": self.path_repo, + # Path to the project exe when it exists + "{[C_TEST_PROJECT_EXE]}": os.path.join(self.path_test, "build", self.vm_int, "src/pgbackrest"), + # Path to source -- used to construct __FILENAME__ tests + "{[C_TEST_PGB_PATH]}": "../" + path_repo_rel, + # Test expect logging + "{[C_TEST_LOG_EXPECT]}": "true" if module.type == TEST_TYPE_UNIT else "false", + # Test log level + "{[C_LOG_LEVEL_TEST]}": log_level_enum(self.log_level), + # Log time/timestamp + "{[C_TEST_TIMING]}": "true" if self.log_time else "false", + # Test architecture + "{[C_TEST_ARCHITECTURE]}": self.architecture, + # Test timezone + "{[C_TEST_TZ]}": ("// No timezone specified" if self.time_zone is None else 'hrnTzSet("%s");' % self.time_zone), + # Scale performance test + "{[C_TEST_SCALE]}": "%u" % self.scale, + # Does this test run in a container? + "{[C_TEST_CONTAINER]}": "true" if self.vm != "none" else "false", + # User/group info + "{[C_TEST_GROUP]}": group_name(), + "{[C_TEST_GROUP_ID]}": "%u" % os.getgid(), + "{[C_TEST_USER]}": user_name(), + "{[C_TEST_USER_LEN]}": "%u" % len(user_name()), + "{[C_TEST_USER_ID]}": "%u" % os.getuid(), + # VM for integration testing + "{[C_TEST_VM]}": self.vm_int, + # PostgreSQL version for integration testing + "{[C_TEST_PG_VERSION]}": self.pg_version, + # Test id + "{[C_TEST_IDX]}": "%u" % self.vm_id, + # Include test file + "{[C_TEST_INCLUDE]}": '#include "%s/test/src/module/%sTest.c"' % (path_repo_rel, bld_enum(None, module.name)), + # Test list + "{[C_TEST_LIST]}": "\n ".join( + "hrnAdd(%3u, %8s);" % (idx + 1, "true" if self.test in (None, idx + 1) else "false") for idx in range(module.total) + ), + # Profiling + "{[C_TEST_PROFILE]}": "true" if self.profile else "false", + "{[C_TEST_PATH_BUILD]}": self.path_unit_build, + } + + for key, value in replace.items(): + test_c = test_c.replace(key, value) + + self._write("test.c", test_c) + + ################################################################################################################################ + def build(self): + """Generate the unit build.""" + + self._file_list = [] + + self._build_shim() + harness_list, harness_include_list = self._build_harness() + self._build_meson(harness_list, harness_include_list) + self._build_test_c(harness_list) + + # Clean files that are not valid for this test + for name in path_list_recurse(self.path_unit): + if name.startswith("build/") or name in self._file_list: + continue + + file_remove(os.path.join(self.path_unit, name), error_on_missing=True) + + +#################################################################################################################################### +# User and group the test runs as. Looked up once since the values are used several times and never change. +_user_name = None +_group_name = None + + +#################################################################################################################################### +def user_name(): + """User the test runs as, looked up once since it is used several times and never changes.""" + + global _user_name + + if _user_name is None: + import pwd + + _user_name = pwd.getpwuid(os.getuid()).pw_name + + return _user_name + + +#################################################################################################################################### +def group_name(): + """Group the test runs as, looked up once since it is used several times and never changes.""" + + global _group_name + + if _group_name is None: + import grp + + _group_name = grp.getgrgid(os.getgid()).gr_name + + return _group_name diff --git a/test/lib/command/test/define.py b/test/lib/command/test/define.py new file mode 100644 index 000000000..31d3a1686 --- /dev/null +++ b/test/lib/command/test/define.py @@ -0,0 +1,318 @@ +"""Parse Define Yaml. + +Keys are looked up by name so only the sequences carry order, which is what YAML guarantees. + +Definitions accumulate as the file is read: a test inherits the coverage, harnesses, shims, and features declared by every test +before it, which is why the module list is built in a single pass over unit, integration, and then performance.""" + +#################################################################################################################################### +import os + +import yaml + +from common.error import TestError, check + +# Test types, in the order they are declared in define.yaml and accumulated +TEST_TYPE_UNIT = "unit" +TEST_TYPE_INTEGRATION = "integration" +TEST_TYPE_PERFORMANCE = "performance" + +TEST_TYPE_LIST = (TEST_TYPE_UNIT, TEST_TYPE_INTEGRATION, TEST_TYPE_PERFORMANCE) + +# Language a test module is written in. A C test is generated and compiled into a binary while a python test is run directly. +TEST_LANG_C = "c" +TEST_LANG_PYTHON = "python" + +TEST_LANG_LIST = (TEST_LANG_C, TEST_LANG_PYTHON) + + +#################################################################################################################################### +class TestDefCoverage: + """A code module covered by a test module.""" + + def __init__(self, name, coverable, included): + self.name = name + self.coverable = coverable # Does this code module include coverable code? + self.included = included # Is this module included in another module? + + +#################################################################################################################################### +class TestDefShimFunction: + """A function that the harness shims.""" + + def __init__(self, name, inc): + self.name = name + self.inc = inc # Included module the function is defined in, or None when it is in the shim module + + +#################################################################################################################################### +class TestDefShim: + """A code module that the harness shims.""" + + def __init__(self, name, integration, function_list): + self.name = name + self.integration = integration # Include in integration tests? + self.function_list = function_list + + +#################################################################################################################################### +class TestDefHarness: + """A harness module.""" + + def __init__(self, name, integration, include_list): + self.name = name + self.integration = integration # Include in integration tests? + self.include_list = include_list # Modules included directly in the harness, in the order they are included + + +#################################################################################################################################### +class TestDefModule: + """A test module.""" + + def __init__(self, name, type): + self.name = name + self.type = type + self.lang = TEST_LANG_C # Language the test module is written in + self.vm_list = [] # Vms the test runs on, empty for all of them + self.total = 0 # Total sub-tests + self.pg_required = False # Is PostgreSQL required? + self.bin_required = False # Is the pgbackrest binary required? + self.container_required = False # Is a container required? + self.flag = None # Compilation flags + self.feature = None # Feature this module introduces + self.feature_list = [] # Features available to this module + self.coverage_list = [] # Code modules covered by this test module + self.depend_list = [] # Code modules this test module depends on + self.include_list = [] # Additional code modules to include in the test module + self.harness_list = [] # Harnesses used by this test + self.shim_list = [] # Shims used by this test + + ################################################################################################################################ + def coverage_find(self, name): + """Find a coverage entry by code module name, or None when the module is not covered.""" + + for coverage in self.coverage_list: + if coverage.name == name: + return coverage + + return None + + ################################################################################################################################ + def shim_find(self, name): + """Find a shim by code module name, or None when the module is not shimmed.""" + + for shim in self.shim_list: + if shim.name == name: + return shim + + return None + + +#################################################################################################################################### +def _parse_shim(shim, harness_include_list, shim_list): + """Parse a single shim entry, either a bare module name or a map with a name and a function list.""" + + # A bare scalar is a module that is shimmed but has no shimmed functions + if isinstance(shim, str): + harness_include_list.append(shim) + + return + + function_list = [] + + for key, value in shim.items(): + if key == "name": + continue + + check(key == "function", "invalid key '%s'" % key) + + for function in value: + # A bare scalar is a function defined in the shim module, else a map naming the included module it is defined in + if isinstance(function, str): + function_list.append(TestDefShimFunction(function, None)) + else: + for name, detail in function.items(): + function_list.append(TestDefShimFunction(name, detail["inc"])) + + check("name" in shim, "shim name is required") + + harness_include_list.append(shim["name"]) + shim_list.append(TestDefShim(shim["name"], True, function_list)) + + +#################################################################################################################################### +def _parse_harness(harness, global_harness_list, global_shim_list): + """Parse a single harness entry, either a bare name or a map with a name, integration flag, and shims.""" + + if isinstance(harness, str): + global_harness_list.append(TestDefHarness(harness, True, [])) + + return + + include_list = [] + shim_list = [] + + for key, value in harness.items(): + if key in ("name", "integration"): + continue + + check(key == "shim", "invalid key '%s'" % key) + + # Shim modules are included in the harness in the order listed + for shim in value: + _parse_shim(shim, include_list, shim_list) + + integration = harness.get("integration", True) + + # Apply the harness integration flag to its shims now that the whole harness has been parsed + for shim in shim_list: + shim.integration = integration + global_shim_list.append(shim) + + global_harness_list.append(TestDefHarness(harness["name"], integration, include_list)) + + +#################################################################################################################################### +def _parse_coverage(coverage_list_raw): + """Parse the coverage list, which holds either a bare code module name or a map tagging it noCode or included.""" + + result = [] + + for coverage in coverage_list_raw: + if isinstance(coverage, str): + entry = TestDefCoverage(coverage, True, False) + else: + for name, type in coverage.items(): + check(type in ("included", "noCode"), "invalid coverage type %s" % type) + + entry = TestDefCoverage(name, type == "included", True) + + result.append(entry) + + return result + + +#################################################################################################################################### +# Parse a single test module +# Keys a module may define. Language and vm are set here when they apply to every test in the module. +_GROUP_KEY_LIST = ("db", "lang", "name", "test", "vm") + +# Keys a test may define. Coverage is applied before depend so the depend list is built in a fixed order no matter how the keys are +# written, since a mapping does not carry order. +_MODULE_KEY_LIST = ( + "binReq", + "containerReq", + "coverage", + "define", + "depend", + "feature", + "harness", + "include", + "lang", + "name", + "total", + "vm", +) + + +#################################################################################################################################### +def _parse_module(test, module, type, state): + """Parse a single test module.""" + + module_name = module["name"] + + for key in test: + check(key in _MODULE_KEY_LIST, "unexpected keyword '%s' in test '%s/%s'" % (key, module_name, test.get("name"))) + + result = TestDefModule("%s/%s" % (module_name, test["name"]), type) + + # Language and vm may be set for the whole module and overridden by a test + result.lang = test.get("lang", module.get("lang", TEST_LANG_C)) + + check(result.lang in TEST_LANG_LIST, "invalid lang '%s' in test '%s'" % (result.lang, result.name)) + + result.vm_list = list(test.get("vm", module.get("vm", []))) + + # Integration tests declare at the module level whether they run against multiple PostgreSQL versions + result.pg_required = module.get("db", False) if type == TEST_TYPE_INTEGRATION else False + result.total = test.get("total", 0) + result.bin_required = test.get("binReq", False) + result.container_required = test.get("containerReq", False) + result.flag = test.get("define") + result.feature = test.get("feature") + result.include_list = list(test.get("include", [])) + + # Each language accumulates its own dependencies. A test may use what it declares plus everything covered by the tests before + # it, which is what makes the module order in this file a documented hierarchy rather than a formality. + depend_state = state["depend"][result.lang] + + if "coverage" in test: + result.coverage_list = _parse_coverage(test["coverage"]) + + for coverage in result.coverage_list: + if coverage.coverable and not coverage.included and coverage.name not in depend_state: + depend_state.append(coverage.name) + + for depend in test.get("depend", []): + if depend not in depend_state: + depend_state.append(depend) + + if "harness" in test: + harness = test["harness"] + + # Harness may be a single entry (bare name or map) or a sequence of entries + for entry in harness if isinstance(harness, list) else [harness]: + _parse_harness(entry, state["harness"], state["shim"]) + + # The depend list is the accumulated list minus anything this module already covers or includes + result.depend_list = [ + depend for depend in depend_state if result.coverage_find(depend) is None and depend not in result.include_list + ] + + # Harnesses and shims accumulate across tests, so this test gets everything declared so far + result.harness_list = [TestDefHarness(h.name, h.integration, list(h.include_list)) for h in state["harness"]] + result.shim_list = [TestDefShim(s.name, s.integration, list(s.function_list)) for s in state["shim"]] + result.feature_list = list(state["feature"]) + + if result.feature is not None: + result.feature = result.feature.upper() + state["feature"].append(result.feature) + + return result + + +#################################################################################################################################### +def test_def_parse(path_repo): + """Parse define.yaml into the list of test modules.""" + + path_define = os.path.join(path_repo, "test/define.yaml") + + with open(path_define, "r") as file: + define = yaml.safe_load(file) + + result = [] + + # Lists that accumulate across every test in the file, in declaration order. Dependencies accumulate per language since a C + # test cannot compile a python module and a python test cannot import a C one. + state = {"depend": {lang: [] for lang in TEST_LANG_LIST}, "feature": [], "harness": [], "shim": []} + + for type in TEST_TYPE_LIST: + for module in define[type]: + for key in module: + check(key in _GROUP_KEY_LIST, "unexpected keyword '%s' in module '%s'" % (key, module.get("name"))) + + for test in module["test"]: + result.append(_parse_module(test, module, type, state)) + + return result + + +#################################################################################################################################### +def test_def_find(module_list, name): + """Find a test module by name.""" + + for module in module_list: + if module.name == name: + return module + + raise TestError("'%s' is not a valid test" % name) diff --git a/test/lib/command/test/python.py b/test/lib/command/test/python.py new file mode 100644 index 000000000..cf7103897 --- /dev/null +++ b/test/lib/command/test/python.py @@ -0,0 +1,134 @@ +"""Python Test Runner. + +Runs a python test module with only the library modules it declared in define.yaml importable. A C unit test compiles in just the +modules it declares, so reaching for anything else fails the build and every cross-dependency ends up documented. Python has no link +step and would happily import anything under test/lib, so an import hook reproduces that. + +This runs as a script rather than inside the harness because the harness has already imported most of the library, and an import +that is already in sys.modules never reaches a hook.""" + +#################################################################################################################################### +import argparse +import importlib.util +import os +import sys + + +#################################################################################################################################### +class ImportGuard: + """Refuse to import a library module the test did not declare.""" + + def __init__(self, path_lib, allow_list): + self.path_lib = path_lib + + # The test harness is support for the tests rather than part of the harness under test, so it is always available + self.allow_list = set(allow_list) | {"harness.test"} + + # Top level packages of the library, i.e. the imports that are guarded. Anything else, e.g. the standard library, is left + # to the finders that follow this one. + self.root_list = {name for name in os.listdir(path_lib) if os.path.isdir(os.path.join(path_lib, name))} + + ################################################################################################################################ + def find_spec(self, fullname, path=None, target=None): + """Refuse a library module the test did not declare, leaving everything else to the finders that follow.""" + + if fullname.split(".")[0] not in self.root_list or fullname in self.allow_list: + return None + + # A package holds no code of its own, and a module is reached through the packages above it, so only modules are declared + if os.path.isdir(os.path.join(self.path_lib, fullname.replace(".", os.sep))): + return None + + raise ImportError( + "'%s' is not declared by this test module -- add it to coverage or depend in define.yaml" % fullname.replace(".", "/") + ) + + +#################################################################################################################################### +def cfg_parse(arg_list): + """Parse the command line.""" + + parser = argparse.ArgumentParser(prog="python.py", description="Run a python test module.") + parser.add_argument("--lib", required=True, metavar="PATH", help="path of the harness library") + parser.add_argument("--test", required=True, metavar="PATH", help="path of the test module") + parser.add_argument("--allow", default="", metavar="LIST", help="comma separated library modules the test may import") + parser.add_argument("--name", default="", metavar="LIST", help="comma separated tests to run, or all of them when not set") + parser.add_argument("--coverage", metavar="PATH", help="measure coverage and write it as json to this path") + + return parser.parse_args(arg_list) + + +#################################################################################################################################### +def coverage_begin(path_lib): + """Start measuring coverage, returning the measurement or None when it cannot be measured. + + Measuring starts before the test module is loaded, since a module imports the code it covers and the definitions in that code + run at import time.""" + + try: + import coverage + except ImportError: + print("unable to measure coverage: no module named 'coverage'") + print("HINT: install python3-coverage (d12, u24, rh10) or py3-coverage (a321, a324), else pip3 install 'coverage>=6.5'") + + return None + + # Branch detail per line arrived in 6.5. An older one reports only summary counts, which cannot say which line to fix. + if tuple(int(part) for part in coverage.__version__.split(".")[:2]) < (6, 5): + print("coverage %s is too old, 6.5 or newer is required for branch detail" % coverage.__version__) + print("HINT: the version packaged for rh9 is too old, so pip3 install 'coverage>=6.5' there") + + return None + + result = coverage.Coverage(branch=True, data_file=None, source=[path_lib]) + result.start() + + return result + + +#################################################################################################################################### +def test_module_run(config): + """Load the test module by path and run it with only the library modules it declared importable.""" + + sys.path.insert(0, config.lib) + sys.meta_path.insert(0, ImportGuard(config.lib, filter(None, config.allow.split(",")))) + + measure = None + + if config.coverage is not None: + measure = coverage_begin(config.lib) + + if measure is None: + return 2 + + spec = importlib.util.spec_from_file_location("test_module", config.test) + test_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(test_module) + + # Imported here rather than at the top so the guard above is installed before the harness is loaded + from harness.test import test_run + + result = 0 if test_run(vars(test_module), list(filter(None, config.name.split(",")))) else 1 + + if measure is not None: + measure.stop() + measure.json_report(outfile=config.coverage) + + return result + + +#################################################################################################################################### +def main(): + """Run the test module named on the command line.""" + + # All output goes to stdout and no bytecode is cached, for the same reasons as test.py + sys.stderr = sys.stdout + sys.dont_write_bytecode = True + + return test_module_run(cfg_parse(sys.argv[1:])) + + +#################################################################################################################################### +# This is the program rather than an import, so it cannot run in the interpreter that is running the tests +if __name__ == "__main__": # {uncoverable_branch} + sys.exit(main()) # {uncoverable} diff --git a/test/lib/command/test/unit.py b/test/lib/command/test/unit.py new file mode 100644 index 000000000..230b095c2 --- /dev/null +++ b/test/lib/command/test/unit.py @@ -0,0 +1,179 @@ +"""Unit Command. + +Prepares the unit test for one test module. A C module is generated into the unit path and compiled there, and the caller runs the +resulting binary so it can wrap it in valgrind. A python module has nothing to build so it is run here. + +The unit path is reused across runs so a rebuild only compiles what changed; a failed build is retried once from a clean path since +a stale build directory is the usual cause.""" + +#################################################################################################################################### +import os +import shutil +import sys + +from command.lint.lint import cmd_lint +from command.test.build import TestBuild +from command.test.define import TEST_LANG_PYTHON, TEST_TYPE_PERFORMANCE, test_def_find, test_def_parse +from common.error import TestError, check +from common.exec import exec_one +from common.log import DETAIL, WARN, log +from common.storage import file_remove, path_create, path_list + + +#################################################################################################################################### +def _arch_fix(arch): + """Normalize the architecture reported by uname to the names the project uses.""" + + if arch == "i686": + return "i386" + + if arch == "arm64": + return "aarch64" + + return arch + + +#################################################################################################################################### +def _path_recreate(path): + """Empty a path, resetting permissions first if needed, and recreate it.""" + + try: + shutil.rmtree(path) + except FileNotFoundError: + pass + except OSError: + # A test may leave paths that cannot be removed until the mode is reset + exec_one("chmod -R 777 '%s'" % path) + shutil.rmtree(path) + + path_create(path, mode=0o770) + + +#################################################################################################################################### +def _python_module(name): + """Convert a code module name to the module it is imported as, e.g. test/common/string_id becomes common.string_id.""" + + check(name.startswith("test/"), "python module '%s' must be under test/" % name) + + return name[len("test/") :].replace("/", ".") + + +#################################################################################################################################### +def _test_python(config, module): + """Run a python test module. + + The test file lives beside the C test modules and is named for the test module, e.g. test/common/string is + test/src/module/test/common/string_test.py. A test file may cover more than one code module. + + It runs in a separate interpreter so the modules it declared are the only library modules it can import. Running it here would + not work since the harness has already imported most of the library and an import found in sys.modules never reaches a hook.""" + + path_test = os.path.join(config.repo_path, "test/src/module", module.name + "_test.py") + + if not os.path.isfile(path_test): + raise TestError("unable to find test module '%s'" % path_test) + + # What the test may import: the code modules it covers, plus what it declared, plus what earlier tests covered + allow_list = [_python_module(coverage.name) for coverage in module.coverage_list if coverage.coverable] + allow_list += [_python_module(name) for name in list(module.depend_list) + list(module.include_list)] + + # A single test can be run while debugging with --test-name, e.g. + # test.py unit test/common/string --test-name=test_string_id_render + path_lib = os.path.join(config.repo_path, "test/lib") + command = "%s '%s/command/test/python.py' --lib='%s' --test='%s' --allow='%s' --name='%s'" % ( + sys.executable, + path_lib, + path_lib, + path_test, + ",".join(allow_list), + config.test_name or "", + ) + + # Coverage is written where the caller asks for it, since the harness runs from a copy of the repository and the report is + # built from the original + if config.coverage and config.coverage_file is not None: + path_create(os.path.dirname(config.coverage_file)) + command += " --coverage='%s'" % config.coverage_file + + # Write the output as it came back rather than through the log, which would indent it as a continuation + print(exec_one(command).rstrip()) + + +#################################################################################################################################### +def cmd_unit(config): + """Prepare the unit test for a test module.""" + + # Linter + cmd_lint(config.repo_path) + + # Find test + module = test_def_find(test_def_parse(config.repo_path), config.module) + + # A python module has nothing to build so run it here + if module.lang == TEST_LANG_PYTHON: + _test_python(config, module) + + return + + # Get test architecture + architecture = config.vm_arch + + if architecture is None: + architecture = _arch_fix(exec_one("uname -m").strip()) + + test_build = TestBuild(config, module, architecture) + path_unit = test_build.path_unit + path_unit_build = test_build.path_unit_build + + # Meson setup depends on the build type, which is release when the module sets a flag, is a performance test, or is profiled + meson_setup = "-Dbuildtype=" + + if module.flag is not None or config.profile or module.type == TEST_TYPE_PERFORMANCE: + if module.flag is not None and module.flag != "-DNDEBUG": + raise TestError("unexpected define '%s'" % module.flag) + + meson_setup += "release" + else: + meson_setup += "debug" + + meson_setup += " -Db_coverage=%s" % ("true" if config.coverage else "false") + + build_retry = False + + while True: + try: + # Generate the unit build + test_build.build() + + if not os.path.exists(os.path.join(path_unit_build, "build.ninja")): + log(DETAIL, "meson setup") + + exec_one("meson setup -Dwerror=true -Dfatal-errors=true %s '%s' '%s'" % (meson_setup, path_unit_build, path_unit)) + # Else reconfigure + else: + log(DETAIL, "meson configure") + + exec_one("meson configure %s '%s'" % (meson_setup, path_unit_build)) + + # Remove old coverage data + path_coverage = os.path.join(path_unit_build, "test-unit.p") + + for name in path_list(path_coverage, expression=r"\.gcda$"): + file_remove(os.path.join(path_coverage, name)) + + # Remove old profile data + file_remove(os.path.join(path_unit_build, "gmon.out")) + + # Ninja build + exec_one("ninja -C '%s'" % path_unit_build) + + break + except TestError as error: + # If this is the first build failure then clean the build path and retry + if build_retry: + raise TestError("build failed for unit %s: %s" % (config.module, error)) + + build_retry = True + + log(WARN, "build failed for unit %s -- will retry: %s" % (config.module, error)) + _path_recreate(path_unit) diff --git a/test/lib/common/error.py b/test/lib/common/error.py new file mode 100644 index 000000000..31ee50e7d --- /dev/null +++ b/test/lib/common/error.py @@ -0,0 +1,28 @@ +"""Error Handling. + +The harness reports failures with TestError, which carries the exit status to return. Exit status mirrors the C harness: 0 is +success, 1 means a top-level coverage check found modules missing coverage, and anything greater is an error.""" + +#################################################################################################################################### +# Exit status for an error. Distinct from 1, which a coverage command uses to report incomplete coverage rather than a failure. +EXIT_ERROR = 2 + + +#################################################################################################################################### +class TestError(Exception): + """A harness error with the exit status to return.""" + + def __init__(self, message, status=EXIT_ERROR): + super().__init__(message) + self.status = status + + +#################################################################################################################################### +def check(condition, message): + """Raise TestError when a condition does not hold. + + Used for invariants the harness itself must maintain, i.e. a failure means the harness or the definitions it reads are wrong + rather than the code under test.""" + + if not condition: + raise TestError(message) diff --git a/test/lib/common/exec.py b/test/lib/common/exec.py new file mode 100644 index 000000000..84dc0ad2c --- /dev/null +++ b/test/lib/common/exec.py @@ -0,0 +1,28 @@ +"""Execute Process. + +Commands run through the shell with stderr folded into stdout, matching execOneExpectP() in src/build/common/exec.c. The combined +output is returned on success and carried in the error on failure, since a failed meson or ninja run says what went wrong there +rather than in its exit status.""" + +#################################################################################################################################### +import subprocess + +from common.error import TestError + + +#################################################################################################################################### +def exec_one(command, result_expect=0): + """Run a command through the shell and return its combined output. + + Raise TestError when the exit status is not the expected one.""" + + process = subprocess.run( + ["sh", "-c", command + " 2>&1"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True + ) + + if process.returncode != result_expect: + output = process.stdout.strip() + + raise TestError("%s terminated unexpectedly [%d]%s%s" % (command, process.returncode, ": " if output else "", output)) + + return process.stdout diff --git a/test/lib/common/log.py b/test/lib/common/log.py new file mode 100644 index 000000000..2885255c5 --- /dev/null +++ b/test/lib/common/log.py @@ -0,0 +1,89 @@ +"""Console Logging. + +Format matches src/common/log.c: an optional timestamp ("YYYY-MM-DD HH:MM:SS.mmm "), the process id ("P00"), and the log level +right-aligned to six characters followed by ": ". Lines after the first in a multi-line message are indented to the prefix width. + +The level is passed to log() rather than being named by the function, as it is in the Perl code, so there is one function to find +rather than one per level. + +All output goes to stdout. The Perl test framework treats anything a test writes to stderr as a failure, so nothing here may go +there, including errors.""" + +#################################################################################################################################### +import sys +import time + +# Log levels, using the same values as LogLevel in src/common/logLevel.h +OFF = 0 +ERROR = 2 +WARN = 3 +INFO = 4 +DETAIL = 5 +DEBUG = 6 +TRACE = 7 + +# Level names as they appear on the command line and in the log +LEVEL_NAME = { + OFF: "off", + ERROR: "error", + WARN: "warn", + INFO: "info", + DETAIL: "detail", + DEBUG: "debug", + TRACE: "trace", +} + +LEVEL_ID = {name: level for level, name in LEVEL_NAME.items()} + +# Current settings, replaced by log_init() +_level = INFO +_timestamp = True + + +#################################################################################################################################### +def log_init(level, timestamp): + """Set the log level and whether timestamps are shown. + + Timestamps are suppressed when generating documentation so the output is reproducible.""" + + global _level, _timestamp + + _level = level + _timestamp = timestamp + + +#################################################################################################################################### +def log_level_parse(name): + """Convert a level name to its id, or None when the name is not valid.""" + + return LEVEL_ID.get(name) + + +#################################################################################################################################### +def log_level_enum(level): + """The C enum name for a level, e.g. "logLevelDebug", used for the test.c substitution.""" + + name = LEVEL_NAME[level] + + return "logLevel" + name[0].upper() + name[1:] + + +#################################################################################################################################### +def log(level, message): + """Write a message at the specified level. + + Continuation lines are indented to line up under the first line.""" + + if level > _level: + return + + prefix = "" + + if _timestamp: + now = time.time() + prefix = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(now)) + ".%03d " % (int(now * 1000) % 1000) + + prefix += "P00 %6s: " % LEVEL_NAME[level].upper() + + sys.stdout.write(prefix + str(message).replace("\n", "\n" + " " * len(prefix)) + "\n") + sys.stdout.flush() diff --git a/test/lib/common/render.py b/test/lib/common/render.py new file mode 100644 index 000000000..a397e7ea4 --- /dev/null +++ b/test/lib/common/render.py @@ -0,0 +1,44 @@ +"""Render Helpers. + +Port of the naming helpers in src/build/common/render.c.""" + +#################################################################################################################################### +from common.string_id import STRING_ID_SEQ_NONE, str_id_from_z, str_id_seq + + +#################################################################################################################################### +def bld_enum(prefix, value): + """Convert a dashed value to an enum name, e.g. "type-convert" becomes "typeConvert". + + When a prefix is given it is prepended and the first letter of the value is upper-cased, e.g. bld_enum("logLevel", "info") + becomes "logLevelInfo".""" + + result = "" + upper = False + + if prefix is not None: + result = prefix + upper = True + + for char in value: + # A dash is removed and the letter that follows it is upper-cased + if char == "-": + upper = True + continue + + result += char.upper() if upper else char + upper = False + + return result + + +#################################################################################################################################### +def bld_str_id_seq(value, sequence=STRING_ID_SEQ_NONE): + """Render the StringId macro for a value, e.g. STRID5("test", 0x2a7250), which is the form the source is expected to contain.""" + + str_id = str_id_from_z(value, sequence) + + if sequence is STRING_ID_SEQ_NONE: + return 'STRID%u("%s", 0x%x)' % ((str_id & 1) + 5, value, str_id) + + return 'STRID%uS("%s", %u, 0x%x)' % ((str_id & 1) + 5, value, str_id_seq(str_id), str_id) diff --git a/test/lib/common/storage.py b/test/lib/common/storage.py new file mode 100644 index 000000000..ffd57d775 --- /dev/null +++ b/test/lib/common/storage.py @@ -0,0 +1,121 @@ +"""File System Helpers. + +Thin wrappers over the standard library that report the affected path in the error, since a bare OSError from deep in a generated +build says nothing about which file the harness was working on.""" + +#################################################################################################################################### +import os +import re + +from common.error import TestError + + +#################################################################################################################################### +def file_read(path, ignore_missing=False): + """Read a file as text. + + Return None instead of raising when the file is missing and ignore_missing is set.""" + + try: + with open(path, "r") as file: + return file.read() + except FileNotFoundError: + if ignore_missing: + return None + + raise TestError("unable to open file '%s' for read" % path) + except OSError as error: + raise TestError("unable to open file '%s' for read: %s" % (path, error.strerror)) + + +#################################################################################################################################### +def file_write(path, content): + """Write a file as text, creating parent paths as needed.""" + + path_create(os.path.dirname(path)) + + try: + with open(path, "w") as file: + file.write(content) + except OSError as error: + raise TestError("unable to open file '%s' for write: %s" % (path, error.strerror)) + + +#################################################################################################################################### +def file_write_differs(path, content): + """Write a file only when the content differs. + + An unchanged generated file then keeps its timestamp and does not trigger a rebuild.""" + + if file_read(path, ignore_missing=True) != content: + file_write(path, content) + + +#################################################################################################################################### +def file_remove(path, error_on_missing=False): + """Remove a file. + + Missing is not an error unless error_on_missing is set.""" + + try: + os.remove(path) + except FileNotFoundError: + if error_on_missing: + raise TestError("unable to remove missing file '%s'" % path) + except OSError as error: + raise TestError("unable to remove file '%s': %s" % (path, error.strerror)) + + +#################################################################################################################################### +def path_create(path, mode=0o750): + """Create a path and any missing parents. + + An existing path is not an error.""" + + if path == "" or os.path.isdir(path): + return + + try: + os.makedirs(path, mode=mode, exist_ok=True) + except OSError as error: + raise TestError("unable to create path '%s': %s" % (path, error.strerror)) + + +#################################################################################################################################### +def path_list(path, expression=None, error_on_missing=False): + """List the names in a path, sorted, optionally filtered by a regular expression. + + A missing path returns an empty list unless error_on_missing is set, since a path listed here is often legitimately absent.""" + + try: + result = sorted(os.listdir(path)) + except FileNotFoundError: + if error_on_missing: + raise TestError("unable to list missing path '%s'" % path) + + return [] + except OSError as error: + raise TestError("unable to list path '%s': %s" % (path, error.strerror)) + + if expression is not None: + regexp = re.compile(expression) + result = [name for name in result if regexp.search(name)] + + return result + + +#################################################################################################################################### +def path_list_recurse(path): + """List files in a path recursively, relative to the path and sorted. + + Directories are not included.""" + + result = [] + + for root, pathList, fileList in os.walk(path): + pathList.sort() + + for name in sorted(fileList): + result.append(os.path.relpath(os.path.join(root, name), path)) + + return sorted(result) diff --git a/test/lib/common/string_id.py b/test/lib/common/string_id.py new file mode 100644 index 000000000..467e087d8 --- /dev/null +++ b/test/lib/common/string_id.py @@ -0,0 +1,120 @@ +"""Represent short strings as integers. + +Port of the encoding in src/common/type/stringId.c. The maps there are written out as 256 byte tables; here they are built from the +character ranges they encode, which is the same mapping.""" + +#################################################################################################################################### +from common.error import TestError + +# Maximum characters that can be encoded with each number of bits +STRID5_MAX = 12 +STRID6_MAX = 10 + +# Sequence value meaning no sequence was specified +STRING_ID_SEQ_NONE = None + +# Sequence at or above this must be stored in the high bits rather than the header +_SEQ_HIGH_MIN = 6 + +# Bits used to encode a character, and the mask for the low sequence bits +_BIT5 = 0 +_BIT6 = 1 +_SEQ_LOW_MASK = 0xE + + +#################################################################################################################################### +def _map_build(bit): + """Build a character to encoding map for the specified number of bits. A character not in the map cannot be encoded.""" + + result = {} + + for index, char in enumerate("abcdefghijklmnopqrstuvwxyz"): + result[char] = index + 1 + + result["-"] = 27 + + # 5-bit encoding has room for only a few digits while 6-bit encoding covers all digits and upper case + if bit == _BIT5: + result["2"] = 28 + result["5"] = 29 + result["6"] = 30 + else: + for index, char in enumerate("0123456789"): + result[char] = index + 28 + + for index, char in enumerate("ABCDEFGHIJKLMNOPQRSTUVWXYZ"): + result[char] = index + 38 + + return result + + +_MAP = {_BIT5: _map_build(_BIT5), _BIT6: _map_build(_BIT6)} + + +#################################################################################################################################### +def _str_id_bit_from_z(bit, value): + """Encode with the specified number of bits, or return 0 when the value does not fit the encoding.""" + + if len(value) > (STRID5_MAX if bit == _BIT5 else STRID6_MAX): + return 0 + + map = _MAP[bit] + + for char in value: + if char not in map: + return 0 + + # Set encoding in the header then shift each character into place + result = bit + shift = 5 if bit == _BIT5 else 6 + + for index, char in enumerate(value): + result |= map[char] << (4 + shift * index) + + return result + + +#################################################################################################################################### +def str_id_from_z(value, sequence=STRING_ID_SEQ_NONE): + """Encode a string as a StringId, adding a sequence when one is specified.""" + + if value == "": + raise TestError("StringId value may not be empty") + + result = _str_id_bit_from_z(_BIT5, value) + + # If 5-bit encoding fails try 6-bit + if result == 0: + result = _str_id_bit_from_z(_BIT6, value) + + # Error when 6-bit encoding also fails + if result == 0: + raise TestError("'%s' contains invalid characters" % value) + + if sequence is not STRING_ID_SEQ_NONE: + # If the sequence fits in the header + if sequence < _SEQ_HIGH_MIN: + result |= (sequence + 1) << 1 + # Else the sequence must be stored in the high bits + else: + bit = result & 1 + + # Shift to remove the bit marker (added back below) and make space for the high sequence + result >>= 1 + result <<= 6 + + result |= (sequence - _SEQ_HIGH_MIN) << 4 | _SEQ_LOW_MASK | bit + + return result + + +#################################################################################################################################### +def str_id_seq(str_id): + """Extract the sequence from a StringId.""" + + sequence = str_id & _SEQ_LOW_MASK + + if sequence != _SEQ_LOW_MASK: + return (sequence >> 1) - 1 + + return ((str_id & 0x1F0) >> 4) + _SEQ_HIGH_MIN diff --git a/test/lib/config/cli.py b/test/lib/config/cli.py new file mode 100644 index 000000000..a00657285 --- /dev/null +++ b/test/lib/config/cli.py @@ -0,0 +1,90 @@ +"""Command Line Interface. + +The commands are the steps the Perl test framework drives: + +unit build the unit test for one test module, run once per test inside its container +coverage merge the coverage produced by the unit tests and write the report +lint run the source linter on its own""" + +#################################################################################################################################### +import argparse + +from common.log import LEVEL_NAME + + +#################################################################################################################################### +def _parser_common(): + """Options shared by every command.""" + + result = argparse.ArgumentParser(add_help=False) + + result.add_argument("--repo-path", default="pgbackrest", metavar="PATH", help="code repository path") + result.add_argument( + "--log-level", default="info", choices=sorted(LEVEL_NAME.values()), metavar="LEVEL", help="console log level" + ) + result.add_argument("--no-log-timestamp", dest="log_timestamp", action="store_false", help="suppress timestamps in the log") + + return result + + +#################################################################################################################################### +def _parser_test_path(parent): + """Options shared by the commands that work in the test path.""" + + result = argparse.ArgumentParser(add_help=False, parents=[parent]) + + result.add_argument("--test-path", default="test", metavar="PATH", help="path where tests are built and run") + result.add_argument("--vm", default="none", metavar="VM", help="vm the test runs on") + + return result + + +#################################################################################################################################### +def cli_parse(arg_list, version): + """Build the parser and parse the command line.""" + + common = _parser_common() + test_path = _parser_test_path(common) + + parser = argparse.ArgumentParser(prog="test.py", description="pgBackRest Test Harness") + parser.add_argument("--version", action="version", version="pgBackRest %s Test Harness" % version) + command = parser.add_subparsers(dest="command", metavar="command", required=True) + + # Unit + # --------------------------------------------------------------------------------------------------------------------------- + unit = command.add_parser( + "unit", parents=[test_path], help="build the unit test for a test module", description="Build a unit test." + ) + unit.add_argument("module", metavar="module/test", help="test module to build") + unit.add_argument("--vm-id", type=int, default=0, metavar="ID", help="0-based id of the vm the test runs on") + unit.add_argument("--vm-arch", metavar="ARCH", help="vm architecture (defaults to the host architecture)") + unit.add_argument("--pg-version", default="invalid", metavar="VERSION", help="pg version for integration tests") + unit.add_argument("--test", type=int, metavar="RUN", help="run only the specified test run") + unit.add_argument("--test-name", metavar="LIST", help="comma separated tests to run in a python test module") + unit.add_argument("--coverage-file", metavar="PATH", help="write coverage for a python test module to this path") + unit.add_argument("--scale", type=int, default=1, metavar="FACTOR", help="scale performance tests") + unit.add_argument("--tz", metavar="TZ", help="run the test in the specified timezone") + unit.add_argument( + "--log-level-test", default="off", choices=sorted(LEVEL_NAME.values()), metavar="LEVEL", help="test log level" + ) + unit.add_argument("--no-coverage", dest="coverage", action="store_false", help="do not build with coverage") + unit.add_argument("--no-back-trace", dest="back_trace", action="store_false", help="do not build with back trace") + unit.add_argument("--optimize", action="store_true", help="build with optimization") + unit.add_argument("--profile", action="store_true", help="build with profiling") + + # Coverage + # --------------------------------------------------------------------------------------------------------------------------- + coverage = command.add_parser( + "coverage", + parents=[test_path], + help="merge coverage and write the report", + description="Merge the coverage produced by the unit tests and write the report.", + ) + coverage.add_argument("module", nargs="+", metavar="module/test", help="test modules that were run") + coverage.add_argument("--coverage-summary", action="store_true", help="write the coverage summary for the documentation") + + # Lint + # --------------------------------------------------------------------------------------------------------------------------- + command.add_parser("lint", parents=[common], help="lint the source", description="Run the source linter.") + + return parser.parse_args(arg_list) diff --git a/test/lib/config/config.py b/test/lib/config/config.py new file mode 100644 index 000000000..866492e73 --- /dev/null +++ b/test/lib/config/config.py @@ -0,0 +1,53 @@ +"""Configuration Load. + +Parses the command line, resolves paths, and initializes logging.""" + +import os +import re + +from common.error import TestError +from common.log import log_init, log_level_parse +from common.storage import file_read +from config.cli import cli_parse + + +#################################################################################################################################### +def project_version(path_repo): + """Read the project version from src/version.h in the repository this script was checked out into. + + The harness always runs from a copy of the repository so the version is never ambiguous.""" + + match = re.search(r'^#define PROJECT_VERSION\s+"([^"]+)"', file_read(os.path.join(path_repo, "src/version.h")), re.M) + + if match is None: + raise TestError("unable to find PROJECT_VERSION in src/version.h") + + return match.group(1) + + +#################################################################################################################################### +def cfg_load(arg_list, path_harness): + """Parse the command line and apply the rules that cannot be expressed in the parser.""" + + config = cli_parse(arg_list, project_version(path_harness)) + + # Make relative paths absolute so the harness does not depend on the working directory + config.repo_path = os.path.abspath(config.repo_path) + + # The repository is only ever read from so it must already exist. Checking here keeps a mistyped path from looking like success + # rather than an error, e.g. the linter would find no files to check and report that everything passed. + if not os.path.isdir(config.repo_path): + raise TestError("repo path '%s' does not exist" % config.repo_path) + + if hasattr(config, "test_path"): + config.test_path = os.path.abspath(config.test_path) + + # Convert log levels to ids now that they are known to be valid + config.log_level = log_level_parse(config.log_level) + + if hasattr(config, "log_level_test"): + config.log_level_test = log_level_parse(config.log_level_test) + + log_init(config.log_level, config.log_timestamp) + + return config diff --git a/test/lib/harness/test.py b/test/lib/harness/test.py new file mode 100644 index 000000000..e1470a4ba --- /dev/null +++ b/test/lib/harness/test.py @@ -0,0 +1,92 @@ +"""Test Harness. + +Support for the harness unit tests, the equivalent of test/src/harness/test.c for the C tests. It is always importable so it does +not need to be declared in define.yaml, and it is not part of the harness itself so it is not covered. + +A test is a module level function named test_*. Running and reporting is unittest's, so a failure is formatted the same way it +always was, but the tests themselves are plain functions with no class and no self. + +The assertions are unittest's own, rebound as plain functions. A bare assert would report only that it failed and not the values, +since that introspection comes from pytest rather than from python.""" + +#################################################################################################################################### +import sys +import unittest + +# Assertions are instance methods so they are taken from an instance that is never run +_case = unittest.TestCase() + +# Comparison +assert_equal = _case.assertEqual +assert_not_equal = _case.assertNotEqual + +# Truth +assert_true = _case.assertTrue +assert_false = _case.assertFalse + +# Identity and membership +assert_is_none = _case.assertIsNone +assert_is_not_none = _case.assertIsNotNone +assert_is_instance = _case.assertIsInstance +assert_in = _case.assertIn +assert_not_in = _case.assertNotIn + +# Errors, used as a context manager, e.g. with assert_raises(TestError) as error: +assert_raises = _case.assertRaises + +# What a test module gets from "from harness.test import *". Listing it here means a test module never has to update its import +# when an assertion is added, while leaving the names resolvable by an editor, which injecting them into the module would not. +__all__ = [ + "assert_equal", + "assert_false", + "assert_in", + "assert_is_instance", + "assert_is_none", + "assert_is_not_none", + "assert_not_equal", + "assert_not_in", + "assert_raises", + "assert_true", +] + + +#################################################################################################################################### +class _FunctionTest(unittest.FunctionTestCase): + """A test function as a test case, reported by function name rather than by the class wrapping it.""" + + ################################################################################################################################ + def __str__(self): + """Report by function name rather than by the class wrapping it.""" + + return self._testFunc.__name__ + + ################################################################################################################################ + def id(self): + """Report by function name rather than by the class wrapping it.""" + + return self._testFunc.__name__ + + ################################################################################################################################ + def shortDescription(self): + """Suppress the description so the runner prints only the name.""" + + return None + + +#################################################################################################################################### +def test_run(namespace, name_list=None): + """Run the test functions in a namespace, in the order they are defined, and return whether they all passed. + + The namespace is the test module, i.e. vars(module) from the runner. Passing name_list runs only those tests.""" + + suite = unittest.TestSuite() + + for name, value in namespace.items(): + # A test is defined in the module rather than imported into it, which keeps an imported helper from being run as a test + if not name.startswith("test_") or not callable(value) or getattr(value, "__module__", None) != namespace["__name__"]: + continue + + if not name_list or name in name_list: + suite.addTest(_FunctionTest(value)) + + return unittest.TextTestRunner(stream=sys.stdout, verbosity=2).run(suite).wasSuccessful() diff --git a/test/lib/pgBackRestTest/Common/ContainerTest.pm b/test/lib/pgBackRestTest/Common/ContainerTest.pm index 881e9ad73..c39c764a9 100644 --- a/test/lib/pgBackRestTest/Common/ContainerTest.pm +++ b/test/lib/pgBackRestTest/Common/ContainerTest.pm @@ -489,10 +489,34 @@ sub containerBuild " libyaml-devel zlib-devel libxml2-devel lz4-devel lz4 bzip2-devel bzip2 perl-JSON-PP ccache meson \\\n" . " libssh2-devel zstd libzstd-devel systemd-devel"; - if ($strOS eq VM_RH9 || $strOS eq VM_RH10) + if ($strOS ne VM_RH8) { $strScript .= " valgrind"; } + + # Coverage for the python tests. rh10 and later package a version new enough while rh9 packages one that predates the + # per line branch detail the report needs, so there it comes from pip below. + if ($strOS eq VM_RH9) + { + $strScript .= " python3-pip"; + } + elsif ($strOS ne VM_RH8) + { + $strScript .= " python3-coverage"; + } + + # The test harness needs a python3 that PyYAML is packaged for. On RH8 that is not the platform python, which meson + # installs and which claims the python3 alternative at a priority nothing else can outrank, so point it explicitly. + if ($strOS eq VM_RH8) + { + $strScript .= + " python3.12 python3.12-pyyaml && \\\n" . + " alternatives --set python3 /usr/bin/python3.12"; + } + else + { + $strScript .= " python3-pyyaml"; + } } elsif ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN) { @@ -504,13 +528,19 @@ sub containerBuild " libyaml-libyaml-perl tzdata devscripts lintian libxml-checker-perl txt2man debhelper \\\n" . " libppi-html-perl libtemplate-perl libtest-differences-perl zlib1g-dev libxml2-dev pkg-config \\\n" . " libbz2-dev bzip2 libyaml-dev libjson-pp-perl liblz4-dev liblz4-tool gnupg lsb-release ccache meson \\\n" . - " libssh2-1-dev libcurl4-openssl-dev libsystemd-dev"; + " libssh2-1-dev libcurl4-openssl-dev libsystemd-dev python3-yaml"; - if ($strOS eq VM_U22 || $strOS eq VM_U24) + if ($strOS ne VM_D12) { $strScript .= " valgrind"; } + # Coverage for the python tests, which are run on u24 and later + if ($strOS ne VM_D12 && $strOS ne VM_U22) + { + $strScript .= " python3-coverage"; + } + $strScript .= ' zstd libzstd-dev'; } elsif ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_ALPINE) @@ -520,7 +550,16 @@ sub containerBuild " apk add --no-cache sudo openssh git rsync tzdata openssh ca-certificates openrc bash && \\\n" . " rc-update add sshd && \\\n" . " apk add --no-cache meson build-base libpq-dev openssl-dev libxml2-dev pkgconfig lz4-dev bzip2-dev\\\n" . - " openssh-keygen zlib-dev yaml-dev libssh2-dev perl perl-yaml-libyaml valgrind lz4 zstd zstd-dev"; + " openssh-keygen zlib-dev yaml-dev libssh2-dev perl perl-yaml-libyaml valgrind lz4 zstd zstd-dev py3-yaml"; + } + + # Coverage for python tests where the packaged version predates the per line branch detail the report needs, so it comes + # from pip rather than from the distribution + if ($strOS eq VM_RH9) + { + $strScript .= sectionHeader() . + "# Install python coverage\n" . + " pip3 install --no-cache-dir 'coverage>=6.5'"; } #--------------------------------------------------------------------------------------------------------------------------- diff --git a/test/lib/pgBackRestTest/Common/DefineTest.pm b/test/lib/pgBackRestTest/Common/DefineTest.pm index 73b776328..850d58ae6 100644 --- a/test/lib/pgBackRestTest/Common/DefineTest.pm +++ b/test/lib/pgBackRestTest/Common/DefineTest.pm @@ -47,6 +47,12 @@ use constant TESTDEF_COVERAGE => 'coverage push @EXPORT, qw(TESTDEF_COVERAGE); use constant TESTDEF_C => 'c'; push @EXPORT, qw(TESTDEF_C); +use constant TESTDEF_LANG => 'lang'; + push @EXPORT, qw(TESTDEF_LANG); +use constant TESTDEF_LANG_PYTHON => 'python'; + push @EXPORT, qw(TESTDEF_LANG_PYTHON); +use constant TESTDEF_PYTHON => 'python'; + push @EXPORT, qw(TESTDEF_PYTHON); use constant TESTDEF_INCLUDE => 'include'; push @EXPORT, qw(TESTDEF_INCLUDE); use constant TESTDEF_INDIVIDUAL => 'individual'; @@ -135,8 +141,13 @@ sub testDefLoad } } - # Set module type variables + # Set module type variables. A python test runs natively like a C test, i.e. it does not require a container, but it + # is not built so it is flagged separately. Language may be set for the whole module and overridden by a test. + my $strLang = coalesce($hModuleTest->{&TESTDEF_LANG}, $hModule->{&TESTDEF_LANG}); + $hTestDefHash->{$strModule}{$strTest}{&TESTDEF_TYPE} = $strModuleType; + $hTestDefHash->{$strModule}{$strTest}{&TESTDEF_PYTHON} = + defined($strLang) && $strLang eq TESTDEF_LANG_PYTHON ? true : false; $hTestDefHash->{$strModule}{$strTest}{&TESTDEF_C} = $strModuleType ne TESTDEF_INTEGRATION && $strTest !~ /perl$/ ? true : false; $hTestDefHash->{$strModule}{$strTest}{&TESTDEF_INTEGRATION} = $strModuleType eq TESTDEF_INTEGRATION ? true : false; diff --git a/test/lib/pgBackRestTest/Common/JobTest.pm b/test/lib/pgBackRestTest/Common/JobTest.pm index 8a66e69c3..014d8cea8 100644 --- a/test/lib/pgBackRestTest/Common/JobTest.pm +++ b/test/lib/pgBackRestTest/Common/JobTest.pm @@ -235,6 +235,11 @@ sub run # Is coverage being tested? my $bCoverage = vmCoverageC($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit}; + # Name of the raw coverage file for this test. A python test name may contain a path separator so flatten it, since the + # coverage command reads the raw files from a single directory. + my $strCoverageName = $self->{oTest}->{&TEST_MODULE} . '-' . $self->{oTest}->{&TEST_NAME}; + $strCoverageName =~ s/\//-/g; + # Create run parameters my $strCommandRunParam = ''; @@ -253,8 +258,8 @@ sub run my $strCommand = ($strVm ne VM_NONE ? "docker exec -i -u ${\TEST_USER} ${strImage} bash -l -c '\\\n" : '') . - $self->{strTestPath} . "/build/${strVm}/test/src/test-pgbackrest" . - ' --repo-path=' . $self->{strTestPath} . '/repo' . ' --test-path=' . $self->{strTestPath} . + 'python3 ' . $self->{strRepoPath} . '/test/test.py unit' . + ' --repo-path=' . $self->{strRepoPath} . ' --test-path=' . $self->{strTestPath} . " --log-level=$self->{strLogLevel}" . ' --vm=' . $self->{oTest}->{&TEST_VM} . (defined($self->{strVmArch}) ? ' --vm-arch=' . $self->{strVmArch} : '') . ' --vm-id=' . $self->{iVmIdx} . ($self->{bProfile} ? ' --profile' : '') . $strCommandRunParam . @@ -262,21 +267,29 @@ sub run ($self->{strTimeZone} ? " --tz='$self->{strTimeZone}'" : '') . ($self->{iScale} ? " --scale=$self->{iScale}" : '') . (defined($self->{oTest}->{&TEST_DB}) ? ' --pg-version=' . $self->{oTest}->{&TEST_DB} : '') . - ($self->{bBackTraceUnit} ? '' : ' --no-back-trace') . ($bCoverage ? '' : ' --no-coverage') . ' test ' . - $self->{oTest}->{&TEST_MODULE} . '/' . $self->{oTest}->{&TEST_NAME} . " && \\\n" . - # Copy stderr to both stderr and stdout so it is displayed and detected as an error. Piping to tee would mask the - # test exit status (the pipe returns tee's status) so save the status to a file and exit with it explicitly. - # Otherwise a test failure (e.g. a valgrind error) would be hidden and reported as missing coverage instead. - "exec 3>&1 && \\\n" . - '{ ' . - # Test with valgrind when requested - ($bValgrind ? - 'valgrind -q --gen-suppressions=all' . - ($self->{oStorageTest}->exists($strValgrindSuppress) ? " --suppressions=${strValgrindSuppress}" : '') . - " --exit-on-first-error=yes --leak-check=full --leak-resolution=high --error-exitcode=25" . ' ' : '') . - "$self->{strUnitPath}/build/test-unit 2>&1 1>&3; echo \$? > $self->{strUnitPath}/result; } | " . - "tee /dev/stderr && \\\n" . - "exit \$(cat $self->{strUnitPath}/result)" . + ($self->{bBackTraceUnit} ? '' : ' --no-back-trace') . ($bCoverage ? '' : ' --no-coverage') . + # A python test writes its own coverage since there is no gcov step for it below + ($self->{oTest}->{&TEST_PYTHON} && $bCoverage ? + ' --coverage-file=' . $self->{strBackRestBase} . '/test/result/coverage/raw/' . + $strCoverageName . '.json' : '') . ' ' . + $self->{oTest}->{&TEST_MODULE} . '/' . $self->{oTest}->{&TEST_NAME} . + # A python test is run by the harness above so there is no binary to run here + ($self->{oTest}->{&TEST_PYTHON} ? '' : + " && \\\n" . + # Copy stderr to both stderr and stdout so it is displayed and detected as an error. Piping to tee would mask + # the test exit status (the pipe returns tee's status) so save the status to a file and exit with it + # explicitly. Otherwise a test failure (e.g. a valgrind error) would be hidden and reported as missing + # coverage instead. + "exec 3>&1 && \\\n" . + '{ ' . + # Test with valgrind when requested + ($bValgrind ? + 'valgrind -q --gen-suppressions=all' . + ($self->{oStorageTest}->exists($strValgrindSuppress) ? " --suppressions=${strValgrindSuppress}" : '') . + " --exit-on-first-error=yes --leak-check=full --leak-resolution=high --error-exitcode=25" . ' ' : '') . + "$self->{strUnitPath}/build/test-unit 2>&1 1>&3; echo \$? > $self->{strUnitPath}/result; } | " . + "tee /dev/stderr && \\\n" . + "exit \$(cat $self->{strUnitPath}/result)") . ($strVm ne VM_NONE ? "'" : ''); my $oExec = new pgBackRestTest::Common::ExecuteTest( @@ -333,7 +346,7 @@ sub end } # If C code generate profile info - if ($iExitStatus == 0 && $self->{oTest}->{&TEST_C} && $self->{bProfile}) + if ($iExitStatus == 0 && $self->{oTest}->{&TEST_C} && !$self->{oTest}->{&TEST_PYTHON} && $self->{bProfile}) { executeTest( ($self->{oTest}->{&TEST_VM} ne VM_NONE ? 'docker exec -i -u ' . TEST_USER . " ${strImage} " : '') . @@ -346,16 +359,20 @@ sub end "$self->{strUnitPath}/gprof.txt", "$self->{strBackRestBase}/test/result/profile/gprof.txt"); } - # If C code generate coverage info - if ($iExitStatus == 0 && $self->{oTest}->{&TEST_C} && vmCoverageC($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit}) + # If C code generate coverage info. A python test wrote its own coverage while it ran. + if ($iExitStatus == 0 && $self->{oTest}->{&TEST_C} && !$self->{oTest}->{&TEST_PYTHON} && + vmCoverageC($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit}) { + my $strCoverageName = $self->{oTest}->{&TEST_MODULE} . '-' . $self->{oTest}->{&TEST_NAME}; + $strCoverageName =~ s/\//-/g; + executeTest( ($self->{oTest}->{&TEST_VM} ne VM_NONE ? 'docker exec -i -u ' . TEST_USER . " ${strImage} " : '') . "gcov --json-format --stdout --branch-probabilities " . (-e "$self->{strUnitPath}/build/test-unit.p" ? "$self->{strUnitPath}/build/test-unit.p" : "$self->{strUnitPath}/build/test-unit\@exe") . '/test.c.gcda > ' . $self->{strBackRestBase} . '/test/result/coverage/raw/' . - $self->{oTest}->{&TEST_MODULE} . '-' . $self->{oTest}->{&TEST_NAME} . '.json'); + $strCoverageName . '.json'); } # Record elapsed time diff --git a/test/lib/pgBackRestTest/Common/ListTest.pm b/test/lib/pgBackRestTest/Common/ListTest.pm index 5686beba9..2d869794f 100644 --- a/test/lib/pgBackRestTest/Common/ListTest.pm +++ b/test/lib/pgBackRestTest/Common/ListTest.pm @@ -44,6 +44,8 @@ use constant TEST_RUN => 'run'; push @EXPORT, qw(TEST_RUN); use constant TEST_VM => 'os'; push @EXPORT, qw(TEST_VM); +use constant TEST_PYTHON => 'python'; + push @EXPORT, qw(TEST_PYTHON); #################################################################################################################################### # testListGet @@ -171,6 +173,7 @@ sub testListGet &TEST_INTEGRATION => $hTest->{&TESTDEF_INTEGRATION}, &TEST_MODULE => $strModule, &TEST_NAME => $strModuleTest, + &TEST_PYTHON => $hTest->{&TESTDEF_PYTHON}, &TEST_RUN => $iTestRunIdx == -1 ? (@{$iyModuleTestRun} == 0 ? undef : $iyModuleTestRun) : [$iTestRunIdx], diff --git a/test/src/build/config/config.yaml b/test/src/build/config/config.yaml deleted file mode 100644 index dfac456d3..000000000 --- a/test/src/build/config/config.yaml +++ /dev/null @@ -1,221 +0,0 @@ -#################################################################################################################################### -# Configuration Definition -#################################################################################################################################### - -#################################################################################################################################### -# Commands -#################################################################################################################################### -command: - help: - log-level-default: DEBUG - parameter-allowed: true - - test: - parameter-allowed: true - - noop: - internal: true - - version: - log-level-default: DEBUG - -#################################################################################################################################### -# Option groups that are not used but must be present for modules to compile -#################################################################################################################################### -optionGroup: - pg: {} - repo: {} - -#################################################################################################################################### -# Options -#################################################################################################################################### -option: - # General options - #--------------------------------------------------------------------------------------------------------------------------------- - back-trace: - type: boolean - default: true - negate: true - command: - test: {} - - buffer-size: - type: size - internal: true - default: 64KiB - allow-list: - - 16KiB - - 32KiB - - 64KiB - - 128KiB - - 256KiB - - 512KiB - - 1MiB - - coverage: - type: boolean - default: true - negate: true - command: - test: {} - - coverage-summary: - type: boolean - default: false - command: - test: {} - - help: - type: boolean - default: false - command: - help: {} - command-role: - main: {} - - neutral-umask: - type: boolean - internal: true - default: true - - optimize: - type: boolean - default: false - command: - test: {} - - output: - type: string-id - sequence: true - command: - version: - default: text - allow-list: - - num - - text - command-role: - main: {} - - pg-version: - type: string - default: invalid - command: - test: {} - - profile: - type: boolean - default: false - command: - test: {} - - repo-path: - type: string - default: pgbackrest - command: - test: {} - - scale: - type: integer - internal: true - default: 1 - allow-range: [1, 1000000000] - command: - test: {} - - test: - type: integer - internal: true - required: false - allow-range: [1, 999] - command: - test: {} - - test-path: - type: string - default: test - command: - test: {} - - tz: - type: string - internal: true - required: false - command: - test: {} - - version: - type: boolean - default: false - command: - help: {} - command-role: - main: {} - - vm: - type: string - internal: true - default: none - command: - test: {} - - vm-arch: - type: string - internal: true - required: false - command: - test: {} - - vm-id: - type: integer - required: false - internal: true - allow-range: [0, 1024] - command: - test: {} - - # Logging options - #--------------------------------------------------------------------------------------------------------------------------------- - log-level: - type: string-id - sequence: true - default: info - allow-list: - - off - - error - - warn - - info - - detail - - debug - - trace - - log-level-test: - inherit: log-level - default: off - - log-timestamp: - type: boolean - default: true - negate: true - command: log-level - - # Options that are not used but must be present for modules to compile. All must have a default or not be required. - #--------------------------------------------------------------------------------------------------------------------------------- - beta: {type: boolean, default: false, command: {noop: {}}} - cmd: {type: string, default-type: dynamic, default: bin, command: {noop: {}}} - compress-level-network: {type: string, required: false, command: {noop: {}}} - config: { - type: string, internal: true, default-type: literal, default: CFGOPTDEF_CONFIG_PATH "/" PROJECT_CONFIG_FILE, negate: true} - config-path: {type: string, required: false, command: {noop: {}}} - config-include-path: { - type: string, default-type: literal, default: CFGOPTDEF_CONFIG_PATH "/" PROJECT_CONFIG_INCLUDE_PATH, command: {noop: {}}} - job-retry: {type: time, default: 0, deprecate: {job-retry-old: {}}, command: {noop: {}}} - job-retry-interval: {type: string, required: false, command: {noop: {}}} - lock: {type: list, required: false, command: {noop: {}}} - log-level-file: {type: string, required: false, command: {noop: {}}} - log-level-stderr: {type: string, required: false, command: {noop: {}}} - pg: {type: string, required: false, command: {noop: {}}} - pg-path: {type: string, required: false, command: {noop: {}}} - repo-type: {type: string, required: false, command: {noop: {}}} - repo: {type: string, required: false, command: {noop: {}}} - spool-path: {type: string, required: false, command: {noop: {}}} - stanza: {type: string, required: false, command: {noop: {}}} diff --git a/test/src/build/help/help.xml b/test/src/build/help/help.xml deleted file mode 100644 index b1c3f6959..000000000 --- a/test/src/build/help/help.xml +++ /dev/null @@ -1,306 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Get help. - - -

Three levels of help are provided. If no command is specified then general help will be displayed. If a command is specified (e.g. test-pgbackrest help test) then a full description of the command will be displayed along with a list of valid options. If an option is specified in addition to a command (e.g. pgbackrest help test vm) then a full description of the option as it applies to the command will be displayed.

- - - - - - - - - - -

- - - Run a test. - - -

By default tests expect to run with the code repository in a subpath named pgbackrest. Tests will be run in a subpath named test.

- -

A test can be run by using the module option, e.g. --module=common/error.

- - - - - Get version. - - -

Displays installed version.

- - - - - - - - - diff --git a/test/src/command/help/meson.build b/test/src/command/help/meson.build deleted file mode 100644 index 83cc41bc7..000000000 --- a/test/src/command/help/meson.build +++ /dev/null @@ -1,17 +0,0 @@ -#################################################################################################################################### -# Generate help -#################################################################################################################################### -test_help_auto_c_inc = custom_target( - 'test_help.auto.c.inc', - output: 'help.auto.c.inc', - depend_files: [ - '../../build/config/config.yaml', - '../../build/help/help.xml', - ], - command: [ - build_code, - 'help', - '@CURRENT_SOURCE_DIR@/../../..', - '@BUILD_ROOT@/test', - ], -) diff --git a/test/src/command/test/build.c b/test/src/command/test/build.c deleted file mode 100644 index 4db2eabb2..000000000 --- a/test/src/command/test/build.c +++ /dev/null @@ -1,792 +0,0 @@ -/*********************************************************************************************************************************** -Test Build Handler -***********************************************************************************************************************************/ -#include - -#include "build/common/render.h" -#include "command/test/build.h" -#include "common/debug.h" -#include "common/log.h" -#include "common/user.h" -#include "storage/posix/storage.h" -#include "version.h" - -/*********************************************************************************************************************************** -Object type -***********************************************************************************************************************************/ -struct TestBuild -{ - TestBuildPub pub; // Publicly accessible variables -}; - -/*********************************************************************************************************************************** -Constants -***********************************************************************************************************************************/ -#define MESON_COMMENT_BLOCK \ - "############################################################################################################################" \ - "########" - -/**********************************************************************************************************************************/ -TestBuild * -testBldNew( - const String *const pathRepo, const String *const pathTest, const String *const vm, const String *const vmInt, - const unsigned int vmId, const String *const pgVersion, const TestDefModule *const module, const unsigned int test, - const uint64_t scale, const LogLevel logLevel, const bool logTime, const String *const timeZone, - const String *const architecture, const bool coverage, const bool profile, const bool optimize, const bool backTrace) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STRING, pathRepo); - FUNCTION_LOG_PARAM(STRING, pathTest); - FUNCTION_LOG_PARAM(STRING, vm); - FUNCTION_LOG_PARAM(STRING, vmInt); - FUNCTION_LOG_PARAM(UINT, vmId); - FUNCTION_LOG_PARAM(STRING, pgVersion); - FUNCTION_LOG_PARAM_P(VOID, module); - FUNCTION_LOG_PARAM(UINT, test); - FUNCTION_LOG_PARAM(UINT64, scale); - FUNCTION_LOG_PARAM(ENUM, logLevel); - FUNCTION_LOG_PARAM(BOOL, logTime); - FUNCTION_LOG_PARAM(STRING, timeZone); - FUNCTION_LOG_PARAM(STRING, architecture); - FUNCTION_LOG_PARAM(BOOL, coverage); - FUNCTION_LOG_PARAM(BOOL, profile); - FUNCTION_LOG_PARAM(BOOL, optimize); - FUNCTION_LOG_PARAM(BOOL, backTrace); - FUNCTION_LOG_END(); - - ASSERT(pathRepo != NULL); - ASSERT(pathTest != NULL); - ASSERT(vm != NULL); - ASSERT(vmInt != NULL); - ASSERT(module != NULL); - ASSERT(scale != 0); - - OBJ_NEW_BEGIN(TestBuild, .childQty = MEM_CONTEXT_QTY_MAX) - { - *this = (TestBuild) - { - .pub = - { - .pathRepo = strDup(pathRepo), - .pathTest = strDup(pathTest), - .vm = strDup(vm), - .vmInt = strDup(vmInt), - .vmId = vmId, - .pgVersion = strDup(pgVersion), - .module = module, - .test = test, - .scale = scale, - .logLevel = logLevel, - .logTime = logTime, - .timeZone = strDup(timeZone), - .architecture = strDup(architecture), - .coverage = coverage, - .profile = profile, - .optimize = optimize, - .backTrace = backTrace, - }, - }; - - this->pub.storageRepo = storagePosixNewP(testBldPathRepo(this)); - this->pub.storageTest = storagePosixNewP(testBldPathTest(this)); - } - OBJ_NEW_END(); - - FUNCTION_LOG_RETURN(TEST_BUILD, this); -} - -/*********************************************************************************************************************************** -Shim functions in a module -***********************************************************************************************************************************/ -static String * -testBldShim(const String *const shimC, const StringList *const functionList) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STRING, shimC); - FUNCTION_LOG_PARAM(STRING_LIST, functionList); - FUNCTION_LOG_END(); - - String *const result = strNew(); - - MEM_CONTEXT_TEMP_BEGIN() - { - const StringList *const inList = strLstNewSplitZ(shimC, "\n"); - ASSERT(strEmpty(strLstGet(inList, strLstSize(inList) - 1))); - - for (unsigned int inIdx = 0; inIdx < strLstSize(inList); inIdx++) - { - const String *const in = strLstGet(inList, inIdx); - - // Check if this line contains a shimmed function - bool found = false; - - for (unsigned int functionIdx = 0; functionIdx < strLstSize(functionList); functionIdx++) - { - const String *const function = strLstGet(functionList, functionIdx); - - if (strBeginsWith(in, function) && strChr(in, '(') == (int)strSize(function)) - { - ASSERT(inIdx > 0); - found = true; - - // For static functions add a forward declaration with the original name so the function can be called from the - // module. For extern functions add a forward declaration with the shimmed name so there is no warning from - // missing-prototypes. - strCatChr(result, ' '); - - if (strBeginsWithZ(strLstGet(inList, inIdx - 1), "static ")) - strCat(result, in); - else - strCatFmt(result, "%s_SHIMMED%s", strZ(function), strZ(strSub(in, strSize(function)))); - - unsigned int scanIdx = inIdx + 1; - - while (true) - { - // In a properly formatted C file the end of the list can never be reached - ASSERT(scanIdx < strLstSize(inList)); - - const String *const scan = strLstGet(inList, scanIdx); - - if (strEqZ(scan, "{")) - break; - - if (strEndsWithZ(strLstGet(inList, scanIdx - 1), ",")) - strCatChr(result, ' '); - - strCat(result, strTrim(strDup(scan))); - scanIdx++; - } - - strCatZ(result, "; "); - strCat(result, strLstGet(inList, inIdx - 1)); - - // Alter the function name so it can be shimmed - strCatFmt(result, "\n%s_SHIMMED%s", strZ(function), strZ(strSub(in, strSize(function)))); - break; - } - } - - // Just copy the line when a function is not found - if (!found) - { - if (inIdx != 0) - strCatChr(result, '\n'); - - strCat(result, in); - } - } - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN(STRING, result); -} - -/*********************************************************************************************************************************** -Write files into the test path and keep a list of files written -***********************************************************************************************************************************/ -static void -testBldWrite(const Storage *const storage, StringList *const fileList, const char *const file, const Buffer *const content) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STORAGE, storage); - FUNCTION_LOG_PARAM(STRING_LIST, fileList); - FUNCTION_LOG_PARAM(STRINGZ, file); - FUNCTION_LOG_PARAM(BUFFER, content); - FUNCTION_LOG_END(); - - MEM_CONTEXT_TEMP_BEGIN() - { - // Only write if the content has changed - const Buffer *const current = storageGetP(storageNewReadP(storage, STR(file), .ignoreMissing = true)); - - if (current == NULL || !bufEq(content, current)) - storagePutP(storageNewWriteP(storage, STR(file)), content); - - strLstAddZ(fileList, file); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN_VOID(); -} - -/*********************************************************************************************************************************** -Generate a relative path from the compare path to the base path -***********************************************************************************************************************************/ -static String * -cmdBldPathModule(const String *const moduleName) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STRING, moduleName); - FUNCTION_LOG_END(); - - String *const result = strNew(); - - MEM_CONTEXT_TEMP_BEGIN() - { - if (strBeginsWithZ(moduleName, "test/")) - strCatFmt(result, "test/src%s", strZ(strSub(moduleName, 4))); - else if (strBeginsWithZ(moduleName, "doc/")) - strCatFmt(result, "doc/src%s", strZ(strSub(moduleName, 3))); - else - strCatFmt(result, "src/%s", strZ(moduleName)); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN(STRING, result); -} - -/*********************************************************************************************************************************** -Generate a relative path from the compare path to the base path -***********************************************************************************************************************************/ -static String * -cmdBldPathRelative(const String *const base, const String *const compare) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STRING, base); - FUNCTION_LOG_PARAM(STRING, compare); - FUNCTION_LOG_END(); - - ASSERT(base != NULL); - ASSERT(compare != NULL); - ASSERT(!strEq(base, compare)); - - String *const result = strNew(); - - MEM_CONTEXT_TEMP_BEGIN() - { - const StringList *const baseList = strLstNewSplitZ(base, "/"); - const StringList *const compareList = strLstNewSplitZ(compare, "/"); - unsigned int compareIdx = 0; - - // Find the part of the paths that is the same - while ( - compareIdx < strLstSize(baseList) && compareIdx < strLstSize(compareList) && - strEq(strLstGet(baseList, compareIdx), strLstGet(compareList, compareIdx))) - { - compareIdx++; - } - - // Generate ../ part of relative path - bool first = true; - - for (unsigned int dotIdx = compareIdx; dotIdx < strLstSize(baseList); dotIdx++) - { - if (!first) - strCatChr(result, '/'); - else - first = false; - - strCatZ(result, ".."); - } - - // Add remaining path - for (unsigned int pathIdx = compareIdx; pathIdx < strLstSize(compareList); pathIdx++) - { - // If first is still set then compare must be a subpath of base (i.e. there were no .. parts) - if (!first) - { - strCatChr(result, '/'); - first = false; - } - - strCat(result, strLstGet(compareList, pathIdx)); - } - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN(STRING, result); -} - -/**********************************************************************************************************************************/ -void -testBldUnit(TestBuild *const this) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(TEST_BUILD, this); - FUNCTION_LOG_END(); - - ASSERT(this != NULL); - - userInit(); - - MEM_CONTEXT_TEMP_BEGIN() - { - const Storage *const storageUnit = storagePosixNewP( - strNewFmt("%s/unit-%u/%s", strZ(testBldPathTest(this)), testBldVmId(this), strZ(testBldVm(this))), .write = true); - const Storage *const storageTestId = storagePosixNewP( - strNewFmt("%s/test-%u", strZ(testBldPathTest(this)), testBldVmId(this)), .write = true); - StringList *const storageUnitList = strLstNew(); - const TestDefModule *const module = testBldModule(this); - const String *const pathUnit = storagePathP(storageUnit, NULL); - const String *const pathRepo = testBldPathRepo(this); - const String *const pathRepoRel = cmdBldPathRelative(pathUnit, pathRepo); - - // Build shim modules - // ------------------------------------------------------------------------------------------------------------------------- - for (unsigned int shimIdx = 0; shimIdx < lstSize(module->shimList); shimIdx++) - { - const TestDefShim *const shim = lstGet(module->shimList, shimIdx); - - // Skip this shim for integration tests - if (module->type == testDefTypeIntegration && !shim->integration) - continue; - - // Build the list of modules to shim. The shim module .c is always shimmed and written since the harness includes it. A - // function may instead be defined in an included .c.inc module, in which case that module is shimmed as well. - StringList *const shimModuleList = strLstNew(); - strLstAdd(shimModuleList, shim->name); - - for (unsigned int funcIdx = 0; funcIdx < lstSize(shim->functionList); funcIdx++) - { - const TestDefShimFunction *const function = lstGet(shim->functionList, funcIdx); - - if (function->inc != NULL) - strLstAddIfMissing(shimModuleList, strNewFmt("%s/%s", strZ(strPath(shim->name)), strZ(function->inc))); - } - - for (unsigned int shimModuleIdx = 0; shimModuleIdx < strLstSize(shimModuleList); shimModuleIdx++) - { - const String *const shimModule = strLstGet(shimModuleList, shimModuleIdx); - - // The first module is the shim module .c; the rest are included .c.inc modules - const bool included = shimModuleIdx != 0; - - // Get the functions to shim in this module - StringList *const functionList = strLstNew(); - - for (unsigned int funcIdx = 0; funcIdx < lstSize(shim->functionList); funcIdx++) - { - const TestDefShimFunction *const function = lstGet(shim->functionList, funcIdx); - const String *const functionModule = - function->inc == NULL ? shim->name : strNewFmt("%s/%s", strZ(strPath(shim->name)), strZ(function->inc)); - - if (strEq(functionModule, shimModule)) - strLstAdd(functionList, function->name); - } - - // Read the source. An included module is a .c.inc, otherwise a .c. - const String *const readFile = strNewFmt("%s.c%s", strZ(cmdBldPathModule(shimModule)), included ? ".inc" : ""); - - String *const shimC = strCatBuf( - strNew(), - storageGetP(storageNewReadP(testBldStorageRepo(this), strNewFmt("%s/%s", strZ(pathRepo), strZ(readFile))))); - - // Write the shimmed source. The shim module .c keeps its normal path. An included .c.inc uses the include path - // so it resolves to this copy (via the unit include dir, before the repo copy) when the shim module includes it. - const String *const writeFile = included ? strNewFmt("%s.c.inc", strZ(shimModule)) : readFile; - - testBldWrite(storageUnit, storageUnitList, strZ(writeFile), BUFSTR(testBldShim(shimC, functionList))); - } - } - - // Build harness modules - // ------------------------------------------------------------------------------------------------------------------------- - StringList *const harnessList = strLstNew(); - StringList *const harnessIncludeList = strLstNew(); - - for (unsigned int harnessIdx = 0; harnessIdx < lstSize(module->harnessList); harnessIdx++) - { - const TestDefHarness *const harness = lstGet(module->harnessList, harnessIdx); - - // Skip this harness for integration tests - if (module->type == testDefTypeIntegration && !harness->integration) - continue; - - const String *const harnessFile = strNewFmt("test/src/harness/%s.c", strZ(harness->name)); - const String *harnessPath = strNewFmt("%s/%s", strZ(pathRepo), strZ(harnessFile)); - - // If there are includes then copy and update the harness - if (!strLstEmpty(harness->includeList)) - { - String *const includeReplace = strNew(); - - for (unsigned int includeIdx = 0; includeIdx < strLstSize(harness->includeList); includeIdx++) - { - const String *const include = strLstGet(harness->includeList, includeIdx); - - strCatFmt( - includeReplace, "%s#include \"%s/%s.c\"", includeIdx == 0 ? "" : "\n", - lstExists(module->shimList, &include) ? strZ(pathUnit) : strZ(pathRepo), strZ(cmdBldPathModule(include))); - - strLstAdd(harnessIncludeList, include); - } - - String *const harnessC = strCatBuf(strNew(), storageGetP(storageNewReadP(testBldStorageRepo(this), harnessPath))); - - strReplace(harnessC, STRDEF("{[SHIM_MODULE]}"), includeReplace); - testBldWrite(storageUnit, storageUnitList, strZ(harnessFile), BUFSTR(harnessC)); - strLstAdd(harnessList, harnessFile); - } - // Else harness can be referenced directly from the repo path - else - strLstAddFmt(harnessList, "%s/%s", strZ(pathRepoRel), strZ(harnessFile)); - } - - // Copy meson_options.txt - // ------------------------------------------------------------------------------------------------------------------------- - testBldWrite( - storageUnit, storageUnitList, "meson_options.txt", - storageGetP(storageNewReadP(testBldStorageRepo(this), STRDEF("meson_options.txt")))); - - // Build meson.build - // ------------------------------------------------------------------------------------------------------------------------- - String *const mesonBuild = strCatBuf( - strNew(), storageGetP(storageNewReadP(testBldStorageRepo(this), STRDEF("meson.build")))); - - // Comment out subdirs that are not used for testing - strReplace(mesonBuild, STRDEF("subdir('"), STRDEF("# subdir('")); - - // Comment out the distribution script, which is only used when building a distribution tarball - strReplace(mesonBuild, STRDEF("meson.add_dist_script("), STRDEF("# meson.add_dist_script(")); - - if (!testBldBackTrace(this)) - { - strReplace( - mesonBuild, STRDEF(" configuration.set('HAVE_LIBBACKTRACE'"), - STRDEF("# configuration.set('HAVE_LIBBACKTRACE'")); - } - - // Write build.auto.in - strCatZ( - mesonBuild, - "\n" - MESON_COMMENT_BLOCK "\n" - "# Write configuration\n" - MESON_COMMENT_BLOCK "\n" - "configure_file(output: 'build.auto.h', configuration: configuration)\n" - "\n" - "add_global_arguments('-DFN_EXTERN=extern', language : 'c')\n" - "add_global_arguments('-DVR_EXTERN_DECLARE=extern', language : 'c')\n" - "add_global_arguments('-DVR_EXTERN_DEFINE=', language : 'c')\n" - "add_global_arguments('-DERROR_MESSAGE_BUFFER_SIZE=131072', language : 'c')\n"); - - // Configure features - if (module->feature != NULL) - strCatFmt(mesonBuild, "add_global_arguments('-DHRN_INTEST_%s', language : 'c')\n", strZ(module->feature)); - - if (module->featureList != NULL) - { - for (unsigned int featureIdx = 0; featureIdx < strLstSize(module->featureList); featureIdx++) - { - strCatFmt( - mesonBuild, "add_global_arguments('-DHRN_FEATURE_%s', language : 'c')\n", - strZ(strLstGet(module->featureList, featureIdx))); - } - } - - // Add compiler flags - if (module->flag != NULL) - strCatFmt(mesonBuild, "add_global_arguments('%s', language : 'c')\n", strZ(module->flag)); - - // Add coverage - if (testBldCoverage(this)) - strCatZ(mesonBuild, "add_global_arguments('-DDEBUG_COVERAGE', language : 'c')\n"); - - // Add container flag - if (!strEqZ(testBldVm(this), "none")) - strCatZ(mesonBuild, "add_global_arguments('-DTEST_CONTAINER_REQUIRED', language : 'c')\n"); - - // Build unit test - strCatZ( - mesonBuild, - "\n" - MESON_COMMENT_BLOCK "\n" - "# Unit test\n" - MESON_COMMENT_BLOCK "\n" - "src_unit = files(\n"); - - for (unsigned int dependIdx = 0; dependIdx < strLstSize(module->dependList); dependIdx++) - { - const String *const depend = strLstGet(module->dependList, dependIdx); - - if (strLstExists(harnessIncludeList, depend)) - continue; - - strCatFmt(mesonBuild, " '%s/%s.c',\n", strZ(pathRepoRel), strZ(cmdBldPathModule(depend))); - } - - // Add harnesses - for (unsigned int harnessIdx = 0; harnessIdx < strLstSize(harnessList); harnessIdx++) - { - const TestDefHarness *const harness = lstGet(module->harnessList, harnessIdx); - - // Add harness depends - const String *const harnessDependPath = strNewFmt("test/src/harness/%s", strZ(harness->name)); - StorageIterator *const storageItr = storageNewItrP( - testBldStorageRepo(this), harnessDependPath, .expression = STRDEF("\\.c$"), .sortOrder = sortOrderAsc); - - while (storageItrMore(storageItr)) - { - strCatFmt( - mesonBuild, " '%s/%s/%s',\n", strZ(pathRepoRel), strZ(harnessDependPath), - strZ(storageItrNext(storageItr).name)); - } - - // Add harness if no includes are in module or coverage includes - unsigned int includeIdx = 0; - - for (; includeIdx < strLstSize(harness->includeList); includeIdx++) - { - const String *const include = strLstGet(harness->includeList, includeIdx); - - if (lstExists(module->coverageList, &include) || strLstExists(module->includeList, include)) - break; - } - - if (includeIdx != strLstSize(harness->includeList)) - continue; - - strCatFmt(mesonBuild, " '%s',\n", strZ(strLstGet(harnessList, harnessIdx))); - } - - strCatFmt( - mesonBuild, - " '%s/test/src/harness/test.c',\n" - " 'test.c',\n" - ")\n" - "\n" - "executable(\n" - " 'test-unit',\n" - " sources: src_unit,\n", - strZ(pathRepoRel)); - - // Add C args - String *const cArg = strNew(); - - if (testBldOptimize(this) || module->type == testDefTypePerformance) - strCatZ(cArg, "\n '-O2',"); - - if (testBldProfile(this)) - { - strCatZ( - cArg, - "\n '-pg'," - "\n '-no-pie',"); - } - - if (!strEmpty(cArg)) - { - strCatFmt( - mesonBuild, - " c_args: [%s\n" - " ],\n", - strZ(cArg)); - } - - // Add linker args - String *const linkArg = strNew(); - - if (testBldProfile(this)) - { - strCatZ( - linkArg, - "\n '-pg'," - "\n '-no-pie',"); - } - - if (!strEmpty(linkArg)) - { - strCatFmt( - mesonBuild, - " link_args: [%s\n" - " ],\n", - strZ(linkArg)); - } - - strCatFmt( - mesonBuild, - " include_directories:\n" - " include_directories(\n" - " '.',\n" - " '%s/src',\n" - " '%s/doc/src',\n" - " '%s/test/src',\n" - " ),\n" - " dependencies: [\n", - strZ(pathRepoRel), strZ(pathRepoRel), strZ(pathRepoRel)); - - if (testBldBackTrace(this)) - { - strCatZ( - mesonBuild, - " lib_backtrace,\n"); - } - - strCatZ( - mesonBuild, - " lib_bz2,\n" - " lib_openssl,\n" - " lib_lz4,\n" - " lib_pq,\n" - " lib_ssh2,\n" - " lib_xml,\n" - " lib_yaml,\n" - " lib_z,\n" - " lib_zstd,\n" - " ],\n" - ")\n"); - - testBldWrite(storageUnit, storageUnitList, "meson.build", BUFSTR(mesonBuild)); - - // Build test.c - // ------------------------------------------------------------------------------------------------------------------------- - String *const testC = strCatBuf( - strNew(), storageGetP(storageNewReadP(testBldStorageRepo(this), STRDEF("test/src/test.c")))); - - // Enable debug test trace - if (!testBldProfile(this) && module->type != testDefTypePerformance) - strReplace(testC, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("#define DEBUG_TEST_TRACE")); - else - strReplace(testC, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("// Debug test trace not enabled")); - - // Files to test/include - StringList *const testIncludeFileList = strLstNew(); - - for (unsigned int coverageIdx = 0; coverageIdx < lstSize(module->coverageList); coverageIdx++) - { - const TestDefCoverage *const coverage = lstGet(module->coverageList, coverageIdx); - - if (coverage->coverable && !coverage->included) - strLstAdd(testIncludeFileList, coverage->name); - } - - for (unsigned int includeIdx = 0; includeIdx < strLstSize(module->includeList); includeIdx++) - strLstAdd(testIncludeFileList, strLstGet(module->includeList, includeIdx)); - - StringList *const testIncludeFile = strLstNew(); - StringList *const harnessIncluded = strLstNew(); - - for (unsigned int testIncludeFileIdx = 0; testIncludeFileIdx < strLstSize(testIncludeFileList); testIncludeFileIdx++) - { - const String *const include = strLstGet(testIncludeFileList, testIncludeFileIdx); - unsigned int harnessIdx = 0; - - for (; harnessIdx < lstSize(module->harnessList); harnessIdx++) - { - const TestDefHarness *const harness = lstGet(module->harnessList, harnessIdx); - - if (strLstExists(harness->includeList, include)) - break; - } - - if (harnessIdx != lstSize(module->harnessList)) - { - if (!strLstExists(harnessIncluded, strLstGet(harnessList, harnessIdx))) - { - strLstAddFmt(testIncludeFile, "#include \"%s\"", strZ(strLstGet(harnessList, harnessIdx))); - strLstAdd(harnessIncluded, strLstGet(harnessList, harnessIdx)); - } - } - else - strLstAddFmt(testIncludeFile, "#include \"%s/%s.c\"", strZ(pathRepoRel), strZ(cmdBldPathModule(include))); - } - - strReplace(testC, STRDEF("{[C_INCLUDE]}"), strLstJoin(testIncludeFile, "\n")); - - // Test path - strReplace(testC, STRDEF("{[C_TEST_PATH]}"), storagePathP(storageTestId, NULL)); - - // Harness data path - const String *const pathHarnessData = strNewFmt("%s/data-%u", strZ(testBldPathTest(this)), testBldVmId(this)); - strReplace(testC, STRDEF("{[C_HRN_PATH]}"), pathHarnessData); - - // Harness repo path - strReplace(testC, STRDEF("{[C_HRN_PATH_REPO]}"), pathRepo); - - // Path to the project exe when it exists - strReplace( - testC, STRDEF("{[C_TEST_PROJECT_EXE]}"), - storagePathP(testBldStorageTest(this), strNewFmt("build/%s/src/" PROJECT_BIN, strZ(testBldVmInt(this))))); - - // Path to source -- used to construct __FILENAME__ tests - strReplace(testC, STRDEF("{[C_TEST_PGB_PATH]}"), strNewFmt("../%s", strZ(pathRepoRel))); - - // Test expect logging - strReplace(testC, STRDEF("{[C_TEST_LOG_EXPECT]}"), module->type == testDefTypeUnit ? TRUE_STR : FALSE_STR); - - // Test log level - strReplace( - testC, STRDEF("{[C_LOG_LEVEL_TEST]}"), - bldEnum("logLevel", strLower(strNewZ(logLevelStr(testBldLogLevel(this)))))); - - // Log time/timestamp - strReplace(testC, STRDEF("{[C_TEST_TIMING]}"), STR(cvtBoolToConstZ(testBldLogTime(this)))); - - // Test architecture - strReplace(testC, STRDEF("{[C_TEST_ARCHITECTURE]}"), testBldArchitecture(this)); - - // Test timezone - strReplace( - testC, STRDEF("{[C_TEST_TZ]}"), - testBldTimeZone(this) == NULL ? - STRDEF("// No timezone specified") : strNewFmt("hrnTzSet(\"%s\");", strZ(testBldTimeZone(this)))); - - // Scale performance test - strReplace(testC, STRDEF("{[C_TEST_SCALE]}"), strNewFmt("%" PRIu64, testBldScale(this))); - - // Does this test run in a container? - strReplace(testC, STRDEF("{[C_TEST_CONTAINER]}"), STR(cvtBoolToConstZ(!strEqZ(testBldVm(this), "none")))); - - // User/group info - strReplace(testC, STRDEF("{[C_TEST_GROUP]}"), groupName()); - strReplace(testC, STRDEF("{[C_TEST_GROUP_ID]}"), strNewFmt("%u", groupId())); - strReplace(testC, STRDEF("{[C_TEST_USER]}"), userName()); - strReplace(testC, STRDEF("{[C_TEST_USER_LEN]}"), strNewFmt("%zu", strSize(userName()))); - strReplace(testC, STRDEF("{[C_TEST_USER_ID]}"), strNewFmt("%u", userId())); - - // VM for integration testing - strReplace(testC, STRDEF("{[C_TEST_VM]}"), testBldVmInt(this)); - - // PostgreSQL version for integration testing - strReplace(testC, STRDEF("{[C_TEST_PG_VERSION]}"), testBldPgVersion(this)); - - // Test id - strReplace(testC, STRDEF("{[C_TEST_IDX]}"), strNewFmt("%u", testBldVmId(this))); - - // Include test file - strReplace( - testC, STRDEF("{[C_TEST_INCLUDE]}"), - strNewFmt("#include \"%s/test/src/module/%sTest.c\"", strZ(pathRepoRel), strZ(bldEnum(NULL, module->name)))); - - // Test list - String *const testList = strNew(); - - for (unsigned int testIdx = 0; testIdx < module->total; testIdx++) - { - if (testIdx != 0) - strCatZ(testList, "\n "); - - strCatFmt( - testList, "hrnAdd(%3u, %8s);", testIdx + 1, - cvtBoolToConstZ(testBldTest(this) == 0 || testBldTest(this) == testIdx + 1)); - } - - strReplace(testC, STRDEF("{[C_TEST_LIST]}"), testList); - - // Profiling - strReplace(testC, STRDEF("{[C_TEST_PROFILE]}"), STR(cvtBoolToConstZ(testBldProfile(this)))); - strReplace(testC, STRDEF("{[C_TEST_PATH_BUILD]}"), strNewFmt("%s/build", strZ(pathUnit))); - - // Write file - testBldWrite(storageUnit, storageUnitList, "test.c", BUFSTR(testC)); - - // Clean files that are not valid for this test - // ------------------------------------------------------------------------------------------------------------------------- - StorageIterator *const storageItr = storageNewItrP(storageUnit, NULL, .recurse = true); - - while (storageItrMore(storageItr)) - { - const StorageInfo info = storageItrNext(storageItr); - - if (info.type != storageTypeFile || strBeginsWithZ(info.name, "build/")) - continue; - - if (!strLstExists(storageUnitList, info.name)) - storageRemoveP(storageUnit, info.name, .errorOnMissing = true); - } - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN_VOID(); -} diff --git a/test/src/command/test/build.h b/test/src/command/test/build.h deleted file mode 100644 index bf5569f49..000000000 --- a/test/src/command/test/build.h +++ /dev/null @@ -1,208 +0,0 @@ -/*********************************************************************************************************************************** -Test Build Handler -***********************************************************************************************************************************/ -#ifndef TEST_COMMAND_TEST_BUILD_H -#define TEST_COMMAND_TEST_BUILD_H - -/*********************************************************************************************************************************** -Object type -***********************************************************************************************************************************/ -typedef struct TestBuild TestBuild; - -#include "build/common/string.h" -#include "command/test/define.h" -#include "common/logLevel.h" -#include "common/type/object.h" -#include "storage/storage.h" - -/*********************************************************************************************************************************** -Constructors -***********************************************************************************************************************************/ -TestBuild *testBldNew( - const String *pathRepo, const String *pathTest, const String *const vm, const String *const vmInt, unsigned int vmId, - const String *pgVersion, const TestDefModule *module, unsigned int test, uint64_t scale, LogLevel logLevel, bool logTime, - const String *timeZone, const String *architecture, bool coverage, bool profile, bool optimize, bool backTrace); - -/*********************************************************************************************************************************** -Getters/Setters -***********************************************************************************************************************************/ -typedef struct TestBuildPub -{ - const String *pathRepo; // Code repository path - const String *pathTest; // Test path - const Storage *storageRepo; // Repository storage - const Storage *storageTest; // Test storage - const String *vm; // Vm to run the test on - const String *vmInt; // Vm to run the test on for integration - unsigned int vmId; // Vm id (0-based) to run the test on - const String *pgVersion; // Pg version to run the test on - const TestDefModule *module; // Module definition - unsigned int test; // Specific test to run (0 if all) - LogLevel logLevel; // Test log level - bool logTime; // Log times/timestamps - uint64_t scale; // Scale performance test - const String *timeZone; // Test in timezone - const String *architecture; // Test architecture - bool coverage; // Generate coverage? - bool profile; // Generate profile report? - bool optimize; // Optimize code? - bool backTrace; // Run with back trace? - TestDef tstDef; // Test definitions -} TestBuildPub; - -// Repository path -FN_INLINE_ALWAYS const String * -testBldPathRepo(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->pathRepo; -} - -// Test path -FN_INLINE_ALWAYS const String * -testBldPathTest(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->pathTest; -} - -// Repository storage -FN_INLINE_ALWAYS const Storage * -testBldStorageRepo(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->storageRepo; -} - -// Test storage -FN_INLINE_ALWAYS const Storage * -testBldStorageTest(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->storageTest; -} - -// Vm -FN_INLINE_ALWAYS const String * -testBldVm(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->vm; -} - -// Vm integration -FN_INLINE_ALWAYS const String * -testBldVmInt(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->vmInt; -} - -// Vm id -FN_INLINE_ALWAYS unsigned int -testBldVmId(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->vmId; -} - -// Pg version -FN_INLINE_ALWAYS const String * -testBldPgVersion(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->pgVersion; -} - -// Test Definition -FN_INLINE_ALWAYS const TestDefModule * -testBldModule(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->module; -} - -// Specific test to run -FN_INLINE_ALWAYS unsigned int -testBldTest(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->test; -} - -// Log level -FN_INLINE_ALWAYS LogLevel -testBldLogLevel(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->logLevel; -} - -// Log time/timestamps -FN_INLINE_ALWAYS bool -testBldLogTime(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->logTime; -} - -// Test in timezone -FN_INLINE_ALWAYS const String * -testBldTimeZone(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->timeZone; -} - -// Test architecture -FN_INLINE_ALWAYS const String * -testBldArchitecture(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->architecture; -} - -// Generate coverage? -FN_INLINE_ALWAYS bool -testBldCoverage(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->coverage; -} - -// Generate profile repo? -FN_INLINE_ALWAYS bool -testBldProfile(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->profile; -} - -// Optimize code? -FN_INLINE_ALWAYS bool -testBldOptimize(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->optimize; -} - -// Scale -FN_INLINE_ALWAYS uint64_t -testBldScale(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->scale; -} - -// Run with back trace? -FN_INLINE_ALWAYS bool -testBldBackTrace(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->backTrace; -} - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -void testBldUnit(TestBuild *this); - -/*********************************************************************************************************************************** -Destructor -***********************************************************************************************************************************/ -FN_INLINE_ALWAYS void -testBuildFree(TestBuild *const this) -{ - objFree(this); -} - -/*********************************************************************************************************************************** -Macros for function logging -***********************************************************************************************************************************/ -#define FUNCTION_LOG_TEST_BUILD_TYPE \ - TestBuild * -#define FUNCTION_LOG_TEST_BUILD_FORMAT(value, buffer, bufferSize) \ - objNameToLog(value, "TestBuild", buffer, bufferSize) - -#endif diff --git a/test/src/command/test/coverage.c b/test/src/command/test/coverage.c deleted file mode 100644 index 5b05a6ac7..000000000 --- a/test/src/command/test/coverage.c +++ /dev/null @@ -1,1177 +0,0 @@ -/*********************************************************************************************************************************** -Coverage Testing and Reporting -***********************************************************************************************************************************/ -#include - -#include "build/common/json.h" -#include "common/debug.h" -#include "common/log.h" -#include "common/regExp.h" -#include "common/type/list.h" - -#include "command/test/coverage.h" -#include "command/test/define.h" -#include "storage/posix/storage.h" - -/*********************************************************************************************************************************** -Object types -***********************************************************************************************************************************/ -typedef struct TestCoverageLine -{ - unsigned int no; // Line number - unsigned int hit; // Total times line was hit - List *branchList; // List of branches -} TestCoverageLine; - -typedef struct TestCoverageFunction -{ - String *name; // Function name - unsigned int lineBegin; // Beginning line of function - unsigned int lineEnd; // Beginning line of function - unsigned int hit; // Total times function was hit -} TestCoverageFunction; - -typedef struct TestCoverageFile -{ - String *name; // File name - List *lineList; // List of lines - List *functionList; // List of functions - - unsigned int functionHit; // Functions hit - unsigned int functionTotal; // Total functions - unsigned int branchHit; // Branches hit - unsigned int branchTotal; // Total branches - unsigned int lineHit; // Lines hit - unsigned int lineTotal; // Total lines -} TestCoverageFile; - -typedef struct TestCoverage -{ - List *fileList; // List of files -} TestCoverage; - -/*********************************************************************************************************************************** -New coverage -***********************************************************************************************************************************/ -static TestCoverage * -testCvgNew(void) -{ - FUNCTION_LOG_VOID(logLevelDebug); - - OBJ_NEW_BEGIN(TestCoverage, .childQty = MEM_CONTEXT_QTY_MAX) - { - *this = (TestCoverage) - { - .fileList = lstNewP(sizeof(TestCoverageFile), .comparator = lstComparatorStr), - }; - } - OBJ_NEW_END(); - - FUNCTION_LOG_RETURN(TEST_COVERAGE, this); -} - -/*********************************************************************************************************************************** -Merge coverage from gcov json -***********************************************************************************************************************************/ -static void -testCvgMerge(const TestCoverage *const this, const String *const json, const StringList *const moduleList) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(TEST_COVERAGE, this); - FUNCTION_LOG_PARAM(STRING, json); - FUNCTION_LOG_PARAM(STRING_LIST, moduleList); - FUNCTION_LOG_END(); - - ASSERT(this != NULL); - ASSERT(json != NULL); - ASSERT(moduleList != NULL); - - MEM_CONTEXT_TEMP_BEGIN() - { - // Read gcov json - const TestCoverage *const merge = testCvgNew(); - JsonRead *const read = jsonReadNew(json); - jsonReadObjectBegin(read); - - while (jsonReadUntil(read, jsonTypeObjectEnd)) - { - const String *const key = jsonReadKey(read); - - if (strEqZ(key, "files")) - { - jsonReadArrayBegin(read); - - while (jsonReadUntil(read, jsonTypeArrayEnd)) - { - TestCoverageFile file = {0}; - bool fileAdd = false; - - MEM_CONTEXT_OBJ_BEGIN(merge->fileList) - { - file.lineList = lstNewP(sizeof(TestCoverageLine)); - file.functionList = lstNewP(sizeof(TestCoverageFunction), .comparator = lstComparatorStr); - } - MEM_CONTEXT_OBJ_END(); - - jsonReadObjectBegin(read); - - while (jsonReadUntil(read, jsonTypeObjectEnd)) - { - const String *const key = jsonReadKey(read); - - if (strEqZ(key, "file")) - { - const String *const fileName = jsonReadStr(read); - - for (unsigned int moduleIdx = 0; moduleIdx < strLstSize(moduleList); moduleIdx++) - { - const String *const module = strLstGet(moduleList, moduleIdx); - - if (strEndsWith(fileName, module)) - { - MEM_CONTEXT_OBJ_BEGIN(merge->fileList) - { - file.name = strDup(module); - } - MEM_CONTEXT_OBJ_END(); - - fileAdd = true; - break; - } - } - } - else if (strEqZ(key, "functions")) - { - jsonReadArrayBegin(read); - - while (jsonReadUntil(read, jsonTypeArrayEnd)) - { - TestCoverageFunction function = {0}; - - jsonReadObjectBegin(read); - - while (jsonReadUntil(read, jsonTypeObjectEnd)) - { - const String *const key = jsonReadKey(read); - - if (strEqZ(key, "name")) - { - MEM_CONTEXT_OBJ_BEGIN(file.functionList) - { - function.name = jsonReadStr(read); - } - MEM_CONTEXT_OBJ_END(); - } - else if (strEqZ(key, "start_line")) - { - function.lineBegin = jsonReadUInt(read); - } - else if (strEqZ(key, "end_line")) - { - function.lineEnd = jsonReadUInt(read); - } - else if (strEqZ(key, "execution_count")) - { - function.hit = jsonReadUInt(read); - } - else - jsonReadSkip(read); - } - - jsonReadObjectEnd(read); - - ASSERT(function.name != NULL); - ASSERT(function.lineBegin != 0); - ASSERT(function.lineEnd != 0); - - lstAdd(file.functionList, &function); - } - - jsonReadArrayEnd(read); - } - else if (strEqZ(key, "lines")) - { - jsonReadArrayBegin(read); - - while (jsonReadUntil(read, jsonTypeArrayEnd)) - { - TestCoverageLine line = {0}; - - jsonReadObjectBegin(read); - - while (jsonReadUntil(read, jsonTypeObjectEnd)) - { - const String *const key = jsonReadKey(read); - - if (strEqZ(key, "branches")) - { - jsonReadArrayBegin(read); - - while (jsonReadUntil(read, jsonTypeArrayEnd)) - { - unsigned int branchHit = 0; - - jsonReadObjectBegin(read); - - while (jsonReadUntil(read, jsonTypeObjectEnd)) - { - if (line.branchList == NULL) - { - MEM_CONTEXT_OBJ_BEGIN(file.lineList) - { - line.branchList = lstNewP(sizeof(unsigned int)); - } - MEM_CONTEXT_OBJ_END(); - } - - const String *const key = jsonReadKey(read); - - if (strEqZ(key, "count")) - branchHit = jsonReadUInt(read); - else - jsonReadSkip(read); - } - - jsonReadObjectEnd(read); - lstAdd(line.branchList, &branchHit); - } - - jsonReadArrayEnd(read); - } - else if (strEqZ(key, "count")) - line.hit = jsonReadUInt(read); - else if (strEqZ(key, "line_number")) - line.no = jsonReadUInt(read); - else - jsonReadSkip(read); - } - - jsonReadObjectEnd(read); - - ASSERT(line.no != 0); - - lstAdd(file.lineList, &line); - } - - jsonReadArrayEnd(read); - } - else - jsonReadSkip(read); - } - - jsonReadObjectEnd(read); - - if (fileAdd) - { - ASSERT(file.name != NULL); - - lstAdd(merge->fileList, &file); - } - } - - jsonReadArrayEnd(read); - } - else - jsonReadSkip(read); - } - - jsonReadObjectEnd(read); - - lstSort(merge->fileList, sortOrderAsc); - - // Merge into coverage - for (unsigned int fileIdx = 0; fileIdx < lstSize(merge->fileList); fileIdx++) - { - const TestCoverageFile *const mergeFile = lstGet(merge->fileList, fileIdx); - const TestCoverageFile *const thisFile = lstFind(this->fileList, &mergeFile->name); - ASSERT(mergeFile != NULL); - - // If the file exists then merge - if (thisFile != NULL) - { - ASSERT(lstSize(thisFile->lineList) == lstSize(mergeFile->lineList)); - ASSERT(lstSize(thisFile->functionList) == lstSize(mergeFile->functionList)); - - for (unsigned int lineIdx = 0; lineIdx < lstSize(thisFile->lineList); lineIdx++) - { - TestCoverageLine *const thisLine = lstGet(thisFile->lineList, lineIdx); - const TestCoverageLine *const mergeLine = lstGet(mergeFile->lineList, lineIdx); - ASSERT(thisLine->no == mergeLine->no); - - thisLine->hit += mergeLine->hit; - - if (thisLine->branchList != NULL) - { - ASSERT(mergeLine->branchList != NULL); - ASSERT(lstSize(thisLine->branchList) == lstSize(mergeLine->branchList)); - - for (unsigned int branchIdx = 0; branchIdx < lstSize(thisLine->branchList); branchIdx++) - { - *(unsigned int *)lstGet(thisLine->branchList, branchIdx) += *(unsigned int *)lstGet( - mergeLine->branchList, branchIdx); - } - } - } - - for (unsigned int functionIdx = 0; functionIdx < lstSize(thisFile->functionList); functionIdx++) - { - TestCoverageFunction *const thisFunction = lstGet(thisFile->functionList, functionIdx); - const TestCoverageFunction *const mergeFunction = lstFind(mergeFile->functionList, &thisFunction->name); - ASSERT(mergeFunction != NULL); - ASSERT(thisFunction->lineBegin == mergeFunction->lineBegin); - ASSERT(thisFunction->lineEnd == mergeFunction->lineEnd); - - thisFunction->hit += mergeFunction->hit; - } - } - // Else add the file - else - { - MEM_CONTEXT_OBJ_BEGIN(this->fileList) - { - const TestCoverageFile file = - { - .name = strDup(mergeFile->name), - .lineList = lstMove(mergeFile->lineList, lstMemContext(this->fileList)), - .functionList = lstMove(mergeFile->functionList, lstMemContext(this->fileList)), - }; - - lstAdd(this->fileList, &file); - } - MEM_CONTEXT_OBJ_END(); - } - } - - lstSort(this->fileList, sortOrderAsc); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN_VOID(); -} - -/*********************************************************************************************************************************** -Coverage report -***********************************************************************************************************************************/ -#define TEST_CVG_HTML_TITLE "pgBackRest Coverage Report" - -// {uncrustify_off - - comment inside string} -#define TEST_CVG_HTML_PRE \ - "\n" \ - "\n" \ - "\n" \ - " " TEST_CVG_HTML_TITLE "\n" \ - " \n" \ - " \n" \ - "\n" \ - "\n" \ - "
" TEST_CVG_HTML_TITLE "
\n" -// {uncrustify_on} - -#define TEST_CVG_HTML_TOC_PRE \ - "
\n" \ - "
LINEBRANCHCODE
' +RPT_BRANCH_COVERED = '' +RPT_BRANCH_UNCOVERED_POST = "' +RPT_CODE_UNCOVERED = ' ' +RPT_LINE_POST = "
...
\n" \ - " \n" \ - " \n" \ - " \n" - -#define TEST_CVG_HTML_TOC_COVERED_PRE \ - " \n" \ - " \n" \ - " \n" - -#define TEST_CVG_HTML_TOC_UNCOVERED_PRE \ - " \n" \ - " \n" \ - " \n" - -#define TEST_CVG_HTML_TOC_POST \ - "
FILE
" - -#define TEST_CVG_HTML_TOC_COVERED_POST \ - "
\n" \ - " " - -#define TEST_CVG_HTML_TOC_UNCOVERED_POST \ - "\n" \ - "
\n" - -#define TEST_CVG_HTML_RPT_PRE \ - "
" - -#define TEST_CVG_HTML_RPT_MID2 \ - "
\n" \ - " \n" \ - " \n" \ - " \n" \ - " \n" \ - " \n" \ - " \n" - -#define TEST_CVG_HTML_RPT_LINE_PRE \ - " \n" \ - " \n" \ - " \n" - -#define TEST_CVG_HTML_RPT_BRANCH_UNCOVERED_PRE \ - "\n" \ - " \n" - -#define TEST_CVG_HTML_RPT_CODE \ - " \n" \ - " \n" - -#define TEST_CVG_HTML_RPT_SKIP \ - " \n" \ - " \n" \ - " \n" - -#define TEST_CVG_HTML_RPT_POST TEST_CVG_HTML_TOC_POST - -#define TEST_CVG_HTML_POST \ - "\n" \ - "\n" - -static String * -testCvgReport(const TestCoverage *const coverage, const String *const pathTest) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(TEST_COVERAGE, coverage); - FUNCTION_LOG_PARAM(STRING, pathTest); - FUNCTION_LOG_END(); - - ASSERT(coverage != NULL); - ASSERT(pathTest != NULL); - - String *const result = strCatZ(strNew(), TEST_CVG_HTML_PRE TEST_CVG_HTML_TOC_PRE); - - // Build table of contents - MEM_CONTEXT_TEMP_BEGIN() - { - for (unsigned int fileIdx = 0; fileIdx < lstSize(coverage->fileList); fileIdx++) - { - const TestCoverageFile *const file = lstGet(coverage->fileList, fileIdx); - const String *const fileName = file->name; - - if (lstEmpty(file->lineList)) - strCatFmt(result, TEST_CVG_HTML_TOC_COVERED_PRE "%s" TEST_CVG_HTML_TOC_COVERED_POST, strZ(fileName)); - else - { - strCatFmt( - result, TEST_CVG_HTML_TOC_UNCOVERED_PRE "%s" TEST_CVG_HTML_TOC_UNCOVERED_MID "%s" - TEST_CVG_HTML_TOC_UNCOVERED_POST, strZ(fileName), strZ(fileName)); - } - } - } - MEM_CONTEXT_TEMP_END(); - - strCatZ(result, TEST_CVG_HTML_TOC_POST); - - // Build files that are missing coverage - for (unsigned int fileIdx = 0; fileIdx < lstSize(coverage->fileList); fileIdx++) - { - MEM_CONTEXT_TEMP_BEGIN() - { - const TestCoverageFile *const file = lstGet(coverage->fileList, fileIdx); - - if (!lstEmpty(file->lineList)) - { - const StringList *const lineTextList = strLstNewSplitZ( - strNewBuf( - storageGetP(storageNewReadP(storagePosixNewP(pathTest), strNewFmt("repo/%s", strZ(file->name))))), "\n"); - unsigned int lineLast = 0; - const String *const fileName = file->name; - - strCatFmt( - result, TEST_CVG_HTML_RPT_PRE "%s" TEST_CVG_HTML_RPT_MID1 "%s" TEST_CVG_HTML_RPT_MID2, - strZ(fileName), strZ(fileName)); - - for (unsigned int lineIdx = 0; lineIdx < lstSize(file->lineList); lineIdx++) - { - const TestCoverageLine *const line = lstGet(file->lineList, lineIdx); - - // Check branch coverage and build report string - bool branchCovered = true; - String *const branchStr = strCatZ(strNew(), "["); - - if (line->branchList != NULL) - { - for (unsigned int branchIdx = 0; branchIdx < lstSize(line->branchList); branchIdx++) - { - if (strSize(branchStr) > 1) - strCatChr(branchStr, ' '); - - if ((*(unsigned int *)lstGet(line->branchList, branchIdx)) == 0) - { - strCatChr(branchStr, '-'); - branchCovered = false; - } - else - strCatChr(branchStr, '+'); - } - } - - strCatChr(branchStr, ']'); - - // Add before context - unsigned int contextBegin = line->no > 5 ? line->no - 5 : 1; - unsigned int contextEnd = line->no > 1 ? line->no - 1 : 0; - - if (lineLast != 0 && contextBegin < lineLast + 1) - contextBegin = lineLast + 1; - - if (lineLast != 0 && contextBegin > lineLast + 1) - { - strCatZ(result, TEST_CVG_HTML_RPT_SKIP); - - for (unsigned int contextIdx = contextBegin; contextIdx <= contextEnd; contextIdx++) - { - const String *const lineText = strTrim(strDup(strLstGet(lineTextList, contextIdx - 1))); - - if (strEmpty(lineText) || strEqZ(lineText, "{") || strEqZ(lineText, "}")) - contextBegin = contextIdx + 1; - else - break; - } - } - - for (unsigned int contextIdx = contextBegin; contextIdx <= contextEnd; contextIdx++) - { - strCatFmt( - result, - TEST_CVG_HTML_RPT_LINE_PRE "%u" TEST_CVG_HTML_RPT_BRANCH_COVERED TEST_CVG_HTML_RPT_CODE "%s" - TEST_CVG_HTML_RPT_LINE_POST, - contextIdx, strZ(strLstGet(lineTextList, contextIdx - 1))); - } - - // Output coverage - strCatFmt(result, TEST_CVG_HTML_RPT_LINE_PRE "%u", line->no); - - if (!branchCovered) - { - strCatFmt( - result, TEST_CVG_HTML_RPT_BRANCH_UNCOVERED_PRE "%s" TEST_CVG_HTML_RPT_BRANCH_UNCOVERED_POST, - strZ(branchStr)); - } - else - strCatZ(result, TEST_CVG_HTML_RPT_BRANCH_COVERED); - - if (line->hit == 0) - strCatZ(result, TEST_CVG_HTML_RPT_CODE_UNCOVERED); - else - strCatZ(result, TEST_CVG_HTML_RPT_CODE); - - strCatFmt(result, "%s" TEST_CVG_HTML_RPT_LINE_POST, strZ(strLstGet(lineTextList, line->no - 1))); - - lineLast = line->no; - - // Add after context - contextBegin = line->no + 1; - contextEnd = line->no <= strLstSize(lineTextList) - 5 ? line->no + 5 : strLstSize(lineTextList); - - if (lineIdx < lstSize(file->lineList) - 1) - { - const TestCoverageLine *const lineNext = lstGet(file->lineList, lineIdx + 1); - - if (contextEnd >= lineNext->no) - contextEnd = lineNext->no - 1; - } - - for (unsigned int contextIdx = contextEnd; contextIdx >= contextBegin; contextIdx--) - { - const String *const lineText = strTrim(strDup(strLstGet(lineTextList, contextIdx - 1))); - - if (strEmpty(lineText) || strEqZ(lineText, "{") || strEqZ(lineText, "}")) - contextEnd = contextIdx - 1; - else - break; - } - - for (unsigned int contextIdx = contextBegin; contextIdx <= contextEnd; contextIdx++) - { - strCatFmt( - result, - TEST_CVG_HTML_RPT_LINE_PRE "%u" TEST_CVG_HTML_RPT_BRANCH_COVERED TEST_CVG_HTML_RPT_CODE "%s" - TEST_CVG_HTML_RPT_LINE_POST, - contextIdx, strZ(strLstGet(lineTextList, contextIdx - 1))); - } - - lineLast = contextEnd; - } - - strCatZ(result, TEST_CVG_HTML_RPT_POST); - } - } - MEM_CONTEXT_TEMP_END(); - } - - // Finalize report - strCatZ(result, TEST_CVG_HTML_POST); - - FUNCTION_LOG_RETURN(STRING, result); -} - -/*********************************************************************************************************************************** -Coverage summary -***********************************************************************************************************************************/ -#define TEST_CVG_SUMMARY \ - "\n" \ - " %s\n" \ - " %s\n" \ - " %s\n" \ - " %s\n" \ - "\n" - -typedef struct TestCvgSummaryModule -{ - const String *name; // Name of coverage module - unsigned int functionHit; // Functions hit - unsigned int functionTotal; // Total functions - unsigned int branchHit; // Branches hit - unsigned int branchTotal; // Total branches - unsigned int lineHit; // Lines hit - unsigned int lineTotal; // Total lines -} TestCvgSummaryModule; - -// Helper to render hit/total and percentage -static String * -testCvgSummaryValue(const unsigned int hit, const unsigned int total) -{ - FUNCTION_LOG_BEGIN(logLevelTrace); - FUNCTION_LOG_PARAM(UINT, hit); - FUNCTION_LOG_PARAM(UINT, total); - FUNCTION_LOG_END(); - - // ASSERT(hit <= total); - - String *const result = strNew(); - - // If total zero - if (total == 0) - { - strCatZ(result, "---"); - } - // Else render value - else - { - MEM_CONTEXT_TEMP_BEGIN() - { - strCatFmt(result, "%u/%u (%s)", hit, total, strZ(strNewPct(hit, total))); - } - MEM_CONTEXT_TEMP_END(); - } - - FUNCTION_LOG_RETURN(STRING, result); -} - -static String * -testCvgSummary(const TestCoverage *const coverage) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(TEST_COVERAGE, coverage); - FUNCTION_LOG_END(); - - ASSERT(coverage != NULL); - - String *const result = strNew(); - - MEM_CONTEXT_TEMP_BEGIN() - { - // Track totals - unsigned int functionHit = 0; - unsigned int functionTotal = 0; - unsigned int branchHit = 0; - unsigned int branchTotal = 0; - unsigned int lineHit = 0; - unsigned int lineTotal = 0; - - // Aggregate modules - List *const moduleList = lstNewP(sizeof(TestCvgSummaryModule), .comparator = lstComparatorStr); - - for (unsigned int fileIdx = 0; fileIdx < lstSize(coverage->fileList); fileIdx++) - { - const TestCoverageFile *const file = lstGet(coverage->fileList, fileIdx); - - // Filter out anything beginning with test/ or doc/ - if (strBeginsWithZ(file->name, "test/") || strBeginsWithZ(file->name, "doc/")) - continue; - - const String *const moduleName = strSub(strPath(file->name), sizeof("src/") - 1); - - TestCvgSummaryModule *module = lstFind(moduleList, &moduleName); - - if (module == NULL) - { - const TestCvgSummaryModule moduleNew = {.name = moduleName}; - lstAdd(moduleList, &moduleNew); - - module = lstFind(moduleList, &moduleName); - ASSERT(module != NULL); - } - - module->functionHit += file->functionHit; - functionHit += file->functionHit; - module->functionTotal += file->functionTotal; - functionTotal += file->functionTotal; - module->branchHit += file->branchHit; - branchHit += file->branchHit; - module->branchTotal += file->branchTotal; - branchTotal += file->branchTotal; - module->lineHit += file->lineHit; - lineHit += file->lineHit; - module->lineTotal += file->lineTotal; - lineTotal += file->lineTotal; - } - - lstSort(moduleList, sortOrderAsc); - - // Output modules - for (unsigned int moduleIdx = 0; moduleIdx < lstSize(moduleList); moduleIdx++) - { - const TestCvgSummaryModule *const module = lstGet(moduleList, moduleIdx); - - strCatFmt( - result, TEST_CVG_SUMMARY "\n", strZ(module->name), - strZ(testCvgSummaryValue(module->functionHit, module->functionTotal)), - strZ(testCvgSummaryValue(module->branchHit, module->branchTotal)), - strZ(testCvgSummaryValue(module->lineHit, module->lineTotal))); - } - - strCatFmt( - result, TEST_CVG_SUMMARY, "TOTAL", strZ(testCvgSummaryValue(functionHit, functionTotal)), - strZ(testCvgSummaryValue(branchHit, branchTotal)), strZ(testCvgSummaryValue(lineHit, lineTotal))); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN(STRING, result); -} - -/**********************************************************************************************************************************/ -int -testCvgGenerate( - const String *const pathRepo, const String *const pathTest, const String *const vm, const bool coverageSummary, - const StringList *const moduleList) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STRING, pathRepo); - FUNCTION_LOG_PARAM(STRING, pathTest); - FUNCTION_LOG_PARAM(STRING, vm); - FUNCTION_LOG_PARAM(BOOL, coverageSummary); - FUNCTION_LOG_PARAM(STRING_LIST, moduleList); - FUNCTION_LOG_END(); - - ASSERT(pathRepo != NULL); - ASSERT(pathTest != NULL); - ASSERT(vm != NULL); - ASSERT(moduleList != NULL); - - int result = 0; - - MEM_CONTEXT_TEMP_BEGIN() - { - // Get test module list - const TestDef testDef = testDefParse(storagePosixNewP(pathRepo)); - - // Build a list of covered code modules - StringList *const coverageList = strLstNew(); - - for (unsigned int moduleIdx = 0; moduleIdx < strLstSize(moduleList); moduleIdx++) - { - const String *const moduleName = strLstGet(moduleList, moduleIdx); - const TestDefModule *const module = lstFind(testDef.moduleList, &moduleName); - CHECK_FMT(ParamInvalidError, module != NULL, "'%s' is not a valid test", strZ(moduleName)); - - for (unsigned int coverageIdx = 0; coverageIdx < lstSize(module->coverageList); coverageIdx++) - { - const TestDefCoverage *const coverage = lstGet(module->coverageList, coverageIdx); - - if (coverage->coverable) - strLstAddIfMissing(coverageList, coverage->name); - } - } - - // Remove code modules that require an untested module for coverage - for (unsigned int moduleIdx = 0; moduleIdx < lstSize(testDef.moduleList); moduleIdx++) - { - const TestDefModule *const module = lstGet(testDef.moduleList, moduleIdx); - - for (unsigned int coverageIdx = 0; coverageIdx < lstSize(module->coverageList); coverageIdx++) - { - const TestDefCoverage *const coverage = lstGet(module->coverageList, coverageIdx); - - if (strLstExists(coverageList, coverage->name) && !strLstExists(moduleList, module->name)) - { - LOG_WARN_FMT("module '%s' did not have all tests run required for coverage", strZ(coverage->name)); - strLstRemove(coverageList, coverage->name); - } - } - } - - // Update code module names as they will appear in coverage - StringList *const coverageModList = strLstNew(); - - for (unsigned int coverageIdx = 0; coverageIdx < strLstSize(coverageList); coverageIdx++) - { - String *coverageName = strLstGet(coverageList, coverageIdx); - - if (strBeginsWithZ(coverageName, "test/")) - coverageName = strNewFmt("test/src/%s.c", strZ(strSub(coverageName, 5))); - else if (strBeginsWithZ(coverageName, "doc/")) - coverageName = strNewFmt("doc/src/%s.c", strZ(strSub(coverageName, 4))); - else - coverageName = strNewFmt("src/%s.c", strZ(coverageName)); - - if (strEndsWithZ(coverageName, ".vendor.c")) - coverageName = strNewFmt("%s.inc", strZ(coverageName)); - else if (strEndsWithZ(coverageName, ".inc.c")) - coverageName = strNewFmt("%s.c.inc", strZ(strSubN(coverageName, 0, strSize(coverageName) - 6))); - - strLstAdd(coverageModList, coverageName); - } - - // Combine gcc json files - const Storage *const storageRepo = storagePosixNewP(pathRepo, .write = true); - const Storage *const storageTest = storagePosixNewP(pathTest); - const String *const pathCoverageRaw = storagePathP(storageRepo, STRDEF("test/result/coverage/raw")); - const StringList *const jsonList = storageListP( - storageRepo, pathCoverageRaw, .errorOnMissing = true, .expression = STRDEF("\\.json$")); - TestCoverage *const coverage = testCvgNew(); - - for (unsigned int jsonIdx = 0; jsonIdx < strLstSize(jsonList); jsonIdx++) - { - MEM_CONTEXT_TEMP_BEGIN() - { - testCvgMerge( - coverage, - strNewBuf( - storageGetP( - storageNewReadP( - storageRepo, strNewFmt("%s/%s", strZ(pathCoverageRaw), strZ(strLstGet(jsonList, jsonIdx)))))), - coverageModList); - } - MEM_CONTEXT_TEMP_END(); - } - - // Coverage exceptions - String *const regExpBranchStr = strNewFmt( - "\\s{4}[A-Z][A-Z0-9_]+\\([^\\?]*\\)|\\s{4}(ASSERT|CHECK|CHECK_FMT|assert|switch\\s)\\(|\\{\\+{0,1}(%s%s)", - coverageSummary ? "uncoverable_branch" : "uncover(ed|able)_branch", strEqZ(vm, "none") ? "|vm_covered" : ""); - RegExp *const regExpBranch = regExpNew(regExpBranchStr); - String *const regExpLineStr = strNewFmt( - "\\{\\+{0,1}(%s%s)[^_]", coverageSummary ? "uncoverable" : "uncover(ed|able)", - strEqZ(vm, "none") ? "|vm_covered" : ""); - RegExp *const regExpLine = regExpNew(regExpLineStr); - RegExp *const regExpLog = regExpNew(STRDEF("\\s+FUNCTION_(LOG|TEST)_(VOID|BEGIN|END|PARAM(|_P|_PP))\\(")); - RegExp *const regExpLogReturn = regExpNew(STRDEF("\\s+FUNCTION_(LOG|TEST)_RETURN_VOID\\(")); - - for (unsigned int fileIdx = 0; fileIdx < lstSize(coverage->fileList); fileIdx++) - { - MEM_CONTEXT_TEMP_BEGIN() - { - unsigned int lineIdx = 0; - const TestCoverageFile *const file = lstGet(coverage->fileList, fileIdx); - const StringList *const lineTextList = strLstNewSplitZ( - strNewBuf(storageGetP(storageNewReadP(storageTest, strNewFmt("repo/%s", strZ(file->name))))), "\n"); - - while (lineIdx < lstSize(file->lineList)) - { - ASSERT(lineIdx < strLstSize(lineTextList)); - TestCoverageLine *const line = lstGet(file->lineList, lineIdx); - - // Remove covered lines for debug logging. These are not very interesting for coverage reporting. - if (line->hit != 0) - { - if (regExpMatch(regExpLog, strLstGet(lineTextList, line->no - 1)) || - (regExpMatch(regExpLogReturn, strLstGet(lineTextList, line->no - 1)) && - strEqZ(strLstGet(lineTextList, line->no), "}"))) - { - lstRemoveIdx(file->lineList, lineIdx); - continue; - } - } - - // If not covered then check for line coverage exceptions - if (line->hit == 0 && regExpMatch(regExpLine, strLstGet(lineTextList, line->no - 1))) - line->hit = 1; - - // If not covered then check for branch coverage exceptions - if (line->branchList != NULL && regExpMatch(regExpBranch, strLstGet(lineTextList, line->no - 1))) - { - // Remove branch coverage so it is not reported - lstFree(line->branchList); - line->branchList = NULL; - } - - lineIdx++; - } - } - MEM_CONTEXT_TEMP_END(); - } - - // Calculate totals - for (unsigned int fileIdx = 0; fileIdx < lstSize(coverage->fileList); fileIdx++) - { - TestCoverageFile *const file = lstGet(coverage->fileList, fileIdx); - - file->functionHit = 0; - file->functionTotal = lstSize(file->functionList); - file->branchHit = 0; - file->branchTotal = 0; - file->lineHit = 0; - file->lineTotal = lstSize(file->lineList); - - for (unsigned int lineIdx = 0; lineIdx < lstSize(file->lineList); lineIdx++) - { - const TestCoverageLine *const line = lstGet(file->lineList, lineIdx); - - if (line->hit > 0) - file->lineHit++; - - if (line->branchList != NULL) - { - for (unsigned int branchIdx = 0; branchIdx < lstSize(line->branchList); branchIdx++) - { - file->branchTotal++; - - if (*(unsigned int *)lstGet(line->branchList, branchIdx) > 0) - file->branchHit++; - } - } - } - - for (unsigned int functionIdx = 0; functionIdx < lstSize(file->functionList); functionIdx++) - { - if (((TestCoverageFunction *)lstGet(file->functionList, functionIdx))->hit > 0) - file->functionHit++; - } - } - - // Write coverage summary - if (coverageSummary) - { - CHECK(OptionInvalidError, !strEqZ(vm, "none"), "coverage summary must be run in vm"); - - storagePutP( - storageNewWriteP(storageRepo, STRDEF("doc/xml/auto/metric-coverage-report.auto.xml")), - BUFSTR(testCvgSummary(coverage))); - } - - // Warn on missing coverage - for (unsigned int fileIdx = 0; fileIdx < lstSize(coverage->fileList); fileIdx++) - { - TestCoverageFile *const file = lstGet(coverage->fileList, fileIdx); - - if (file->lineHit != file->lineTotal || file->branchHit != file->branchTotal) - { - LOG_WARN_FMT( - "module '%s' is not fully covered (%u/%u lines, %u/%u branches)", strZ(file->name), file->lineHit, - file->lineTotal, file->branchHit, file->branchTotal); - result++; - } - } - - // Filter out covered lines/branches - for (unsigned int fileIdx = 0; fileIdx < lstSize(coverage->fileList); fileIdx++) - { - const TestCoverageFile *const file = lstGet(coverage->fileList, fileIdx); - unsigned int lineIdx = 0; - - while (lineIdx < lstSize(file->lineList)) - { - const TestCoverageLine *const line = lstGet(file->lineList, lineIdx); - bool covered = line->hit != 0; - - if (covered && line->branchList != NULL) - { - for (unsigned int branchIdx = 0; branchIdx < lstSize(line->branchList); branchIdx++) - { - if ((*(unsigned int *)lstGet(line->branchList, branchIdx)) == 0) - { - covered = false; - break; - } - } - } - - // If line is covered remove it from the report - if (covered) - { - lstRemoveIdx(file->lineList, lineIdx); - } - // Else check the next line - else - lineIdx++; - } - } - - // Write coverage report - storagePutP( - storageNewWriteP(storageRepo, STRDEF("test/result/coverage/coverage.html")), - BUFSTR(testCvgReport(coverage, pathTest))); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN(INT, result); -} diff --git a/test/src/command/test/coverage.h b/test/src/command/test/coverage.h deleted file mode 100644 index 0939df0b6..000000000 --- a/test/src/command/test/coverage.h +++ /dev/null @@ -1,24 +0,0 @@ -/*********************************************************************************************************************************** -Coverage Testing and Reporting -***********************************************************************************************************************************/ -#ifndef TEST_COMMAND_TEST_COVERAGE_H -#define TEST_COMMAND_TEST_COVERAGE_H - -#include "common/type/stringList.h" - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Generate coverage summary/report -int testCvgGenerate( - const String *repoPath, const String *pathTest, const String *vm, bool coverageSummary, const StringList *moduleList); - -/*********************************************************************************************************************************** -Macros for function logging -***********************************************************************************************************************************/ -#define FUNCTION_LOG_TEST_COVERAGE_TYPE \ - TestCoverage * -#define FUNCTION_LOG_TEST_COVERAGE_FORMAT(value, buffer, bufferSize) \ - objNameToLog(value, "TestCoverage", buffer, bufferSize) - -#endif diff --git a/test/src/command/test/define.c b/test/src/command/test/define.c deleted file mode 100644 index 381fa689b..000000000 --- a/test/src/command/test/define.c +++ /dev/null @@ -1,462 +0,0 @@ -/*********************************************************************************************************************************** -Parse Define Yaml -***********************************************************************************************************************************/ -#include - -#include - -#include "common/debug.h" -#include "common/log.h" -#include "storage/posix/storage.h" - -#include "build/common/yaml.h" -#include "command/test/define.h" - -/*********************************************************************************************************************************** -Parse a single shim entry (either a bare module name or a map with name/function) -***********************************************************************************************************************************/ -static void -testDefParseShim(Yaml *const yaml, StringList *const harnessIncludeList, List *const shimList) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(YAML, yaml); - FUNCTION_LOG_PARAM(STRING_LIST, harnessIncludeList); - FUNCTION_LOG_PARAM(LIST, shimList); - FUNCTION_LOG_END(); - - FUNCTION_AUDIT_HELPER(); - - // A bare scalar is a module that is shimmed but has no shimmed functions - if (yamlEventPeekIs(yaml, yamlEventTypeScalar)) - { - strLstAdd(harnessIncludeList, yamlScalarNext(yaml).value); - } - // Else a map with the module name and the functions to shim - else - { - TestDefShim testDefShim = {0}; - List *const functionList = lstNewP(sizeof(TestDefShimFunction)); - - YAML_MAP_BEGIN(yaml) - { - const String *const key = yamlScalarNext(yaml).value; - - if (strEqZ(key, "name")) - { - testDefShim.name = yamlScalarNext(yaml).value; - strLstAdd(harnessIncludeList, testDefShim.name); - } - else - { - CHECK_FMT(AssertError, strEqZ(key, "function"), "invalid key '%s'", strZ(key)); - - YAML_SEQ_BEGIN(yaml) - { - TestDefShimFunction function = {0}; - - // A bare scalar is a function defined in the shim module - if (yamlEventPeekIs(yaml, yamlEventTypeScalar)) - { - function.name = yamlScalarNext(yaml).value; - } - // Else a map, e.g. "- func: {inc: mod}", where the function is defined in the included module - else - { - YAML_MAP_BEGIN(yaml) - { - function.name = yamlScalarNext(yaml).value; - - YAML_MAP_BEGIN(yaml) - { - yamlScalarNextCheckZ(yaml, "inc"); - function.inc = yamlScalarNext(yaml).value; - } - YAML_MAP_END(); - } - YAML_MAP_END(); - } - - // Copy the strings into the list context since they are referenced after parsing - MEM_CONTEXT_OBJ_BEGIN(functionList) - { - function.name = strDup(function.name); - function.inc = strDup(function.inc); - } - MEM_CONTEXT_OBJ_END(); - - lstAdd(functionList, &function); - } - YAML_SEQ_END(); - } - } - YAML_MAP_END(); - - CHECK(AssertError, testDefShim.name != NULL, "shim name is required"); - - testDefShim.functionList = functionList; - lstAdd(shimList, &testDefShim); - } - - FUNCTION_LOG_RETURN_VOID(); -} - -/*********************************************************************************************************************************** -Parse a single harness entry (either a bare name or a map with name/integration/shim) -***********************************************************************************************************************************/ -static void -testDefParseHarness(Yaml *const yaml, List *const globalHarnessList, List *const globalShimList) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(YAML, yaml); - FUNCTION_LOG_PARAM(LIST, globalHarnessList); - FUNCTION_LOG_PARAM(LIST, globalShimList); - FUNCTION_LOG_END(); - - FUNCTION_AUDIT_HELPER(); - - TestDefHarness testDefHarness = {.integration = true}; - StringList *harnessIncludeList = strLstNew(); - List *const shimList = lstNewP(sizeof(TestDefShim)); - - if (yamlEventPeekIs(yaml, yamlEventTypeScalar)) - { - testDefHarness.name = yamlScalarNext(yaml).value; - } - else - { - YAML_MAP_BEGIN(yaml) - { - const String *const type = yamlScalarNext(yaml).value; - - if (strEqZ(type, "name")) - { - testDefHarness.name = yamlScalarNext(yaml).value; - } - else if (strEqZ(type, "integration")) - { - testDefHarness.integration = yamlBoolParse(yamlScalarNext(yaml)); - } - else - { - CHECK_FMT(AssertError, strEqZ(type, "shim"), "invalid key '%s'", strZ(type)); - - // Shim modules are included in the harness in the order listed - YAML_SEQ_BEGIN(yaml) - { - testDefParseShim(yaml, harnessIncludeList, shimList); - } - YAML_SEQ_END(); - } - } - YAML_MAP_END(); - } - - // Apply the harness integration flag to its shims. This is done after the harness has been parsed so the flag applies no matter - // what order the keys appear in. - for (unsigned int shimIdx = 0; shimIdx < lstSize(shimList); shimIdx++) - { - TestDefShim *const testDefShim = lstGet(shimList, shimIdx); - - testDefShim->integration = testDefHarness.integration; - lstAdd(globalShimList, testDefShim); - } - - testDefHarness.includeList = harnessIncludeList; - lstAdd(globalHarnessList, &testDefHarness); - - FUNCTION_LOG_RETURN_VOID(); -} - -/*********************************************************************************************************************************** -Parse module list -***********************************************************************************************************************************/ -static void -testDefParseModuleList(Yaml *const yaml, List *const moduleList) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(YAML, yaml); - FUNCTION_LOG_PARAM(LIST, moduleList); - FUNCTION_LOG_END(); - - FUNCTION_AUDIT_HELPER(); - - // Global lists to be copied to next test - StringList *const globalDependList = strLstNew(); - StringList *const globalFeatureList = strLstNew(); - List *const globalHarnessList = lstNewP(sizeof(TestDefHarness), .comparator = lstComparatorStr); - List *const globalShimList = lstNewP(sizeof(TestDefShim), .comparator = lstComparatorStr); - const char *typeList[] = {"unit", "integration", "performance"}; - - for (TestDefType type = 0; type < LENGTH_OF(typeList); type++) - { - yamlScalarNextCheckZ(yaml, typeList[type]); - - // Module list - YAML_SEQ_BEGIN(yaml) - { - // Module - YAML_MAP_BEGIN(yaml) - { - // Module name - yamlScalarNextCheckZ(yaml, "name"); - const String *const moduleName = yamlScalarNext(yaml).value; - - // Check if next is db for integration tests - bool pgRequired = false; - - if (type == testDefTypeIntegration && strEqZ(yamlEventPeek(yaml).value, "db")) - { - yamlScalarNextCheckZ(yaml, "db"); - pgRequired = yamlBoolParse(yamlScalarNext(yaml)); - } - - // Submodule List - yamlScalarNextCheckZ(yaml, "test"); - - YAML_SEQ_BEGIN(yaml) - { - TestDefModule testDefModule = {.type = type, .pgRequired = pgRequired}; - StringList *const includeList = strLstNew(); - List *const coverageList = lstNewP(sizeof(TestDefCoverage), .comparator = lstComparatorStr); - - // Submodule - YAML_MAP_BEGIN(yaml) - { - YamlEvent subModuleDef = yamlEventNext(yaml); - - if (strEqZ(subModuleDef.value, "binReq")) - { - testDefModule.binRequired = yamlBoolParse(yamlScalarNext(yaml)); - } - else if (strEqZ(subModuleDef.value, "containerReq")) - { - testDefModule.containerRequired = yamlBoolParse(yamlScalarNext(yaml)); - } - else if (strEqZ(subModuleDef.value, "coverage")) - { - YAML_SEQ_BEGIN(yaml) - { - TestDefCoverage testDefCoverage = {0}; - - if (yamlEventPeekIs(yaml, yamlEventTypeScalar)) - { - testDefCoverage.name = yamlScalarNext(yaml).value; - testDefCoverage.coverable = true; - } - else - { - YAML_MAP_BEGIN(yaml) - { - testDefCoverage.name = yamlScalarNext(yaml).value; - const String *const type = yamlScalarNext(yaml).value; - testDefCoverage.included = true; - - if (strEqZ(type, "included")) - { - testDefCoverage.coverable = true; - } - else - CHECK_FMT(AssertError, strEqZ(type, "noCode"), "invalid coverage type %s", strZ(type)); - } - YAML_MAP_END(); - } - - MEM_CONTEXT_OBJ_BEGIN(coverageList) - { - testDefCoverage.name = strDup(testDefCoverage.name); - lstAdd(coverageList, &testDefCoverage); - } - MEM_CONTEXT_OBJ_END(); - - // Also add to the global depend list - if (testDefCoverage.coverable && !testDefCoverage.included) - strLstAddIfMissing(globalDependList, testDefCoverage.name); - } - YAML_SEQ_END(); - } - else if (strEqZ(subModuleDef.value, "define")) - { - testDefModule.flag = yamlScalarNext(yaml).value; - } - else if (strEqZ(subModuleDef.value, "depend")) - { - YAML_SEQ_BEGIN(yaml) - { - strLstAddIfMissing(globalDependList, yamlEventNext(yaml).value); - } - YAML_SEQ_END(); - } - else if (strEqZ(subModuleDef.value, "feature")) - { - testDefModule.feature = yamlScalarNext(yaml).value; - } - else if (strEqZ(subModuleDef.value, "harness")) - { - // Harness may be a single entry (bare name or map) or a sequence of entries - if (yamlEventPeekIs(yaml, yamlEventTypeSeqBegin)) - { - YAML_SEQ_BEGIN(yaml) - { - testDefParseHarness(yaml, globalHarnessList, globalShimList); - } - YAML_SEQ_END(); - } - else - testDefParseHarness(yaml, globalHarnessList, globalShimList); - } - else if (strEqZ(subModuleDef.value, "include")) - { - YAML_SEQ_BEGIN(yaml) - { - strLstAdd(includeList, yamlEventNext(yaml).value); - } - YAML_SEQ_END(); - } - else if (strEqZ(subModuleDef.value, "name")) - { - testDefModule.name = strNewFmt("%s/%s", strZ(moduleName), strZ(yamlScalarNext(yaml).value)); - } - else - { - CHECK_FMT( - FormatError, strEqZ(subModuleDef.value, "total"), - "unexpected keyword '%s' at line %zu, column %zu", - strZ(subModuleDef.value), subModuleDef.line, subModuleDef.column); - - testDefModule.total = cvtZToUInt(strZ(yamlScalarNext(yaml).value)); - } - } - YAML_MAP_END(); - - // Depend list is the global list minus the coverage and include lists - StringList *const dependList = strLstNew(); - - for (unsigned int dependIdx = 0; dependIdx < strLstSize(globalDependList); dependIdx++) - { - const String *const depend = strLstGet(globalDependList, dependIdx); - - if (!lstExists(coverageList, &depend) && !strLstExists(includeList, depend)) - strLstAdd(dependList, depend); - } - - // Add test module - MEM_CONTEXT_OBJ_BEGIN(moduleList) - { - testDefModule.name = strDup(testDefModule.name); - testDefModule.coverageList = lstMove(coverageList, memContextCurrent()); - testDefModule.flag = strDup(testDefModule.flag); - testDefModule.includeList = strLstMove(includeList, memContextCurrent()); - - if (strLstSize(dependList) > 0) - testDefModule.dependList = strLstMove(dependList, memContextCurrent()); - - if (testDefModule.feature != NULL) - testDefModule.feature = strUpper(strDup(testDefModule.feature)); - - if (strLstSize(globalFeatureList) > 0) - testDefModule.featureList = strLstDup(globalFeatureList); - - // Copy harness list - List *const harnessList = lstNewP(sizeof(TestDefHarness), .comparator = lstComparatorStr); - - MEM_CONTEXT_OBJ_BEGIN(harnessList) - { - for (unsigned int harnessIdx = 0; harnessIdx < lstSize(globalHarnessList); harnessIdx++) - { - const TestDefHarness *const globalHarness = lstGet(globalHarnessList, harnessIdx); - const TestDefHarness testDefHarness = - { - .name = strDup(globalHarness->name), - .integration = globalHarness->integration, - .includeList = strLstDup(globalHarness->includeList), - }; - - lstAdd(harnessList, &testDefHarness); - } - } - MEM_CONTEXT_OBJ_END(); - - testDefModule.harnessList = harnessList; - - // Copy shim list - List *const shimList = lstNewP(sizeof(TestDefShim), .comparator = lstComparatorStr); - - MEM_CONTEXT_OBJ_BEGIN(shimList) - { - for (unsigned int shimIdx = 0; shimIdx < lstSize(globalShimList); shimIdx++) - { - const TestDefShim *const globalShim = lstGet(globalShimList, shimIdx); - - // Copy the shim function list - List *const functionList = lstNewP(sizeof(TestDefShimFunction)); - - for (unsigned int funcIdx = 0; funcIdx < lstSize(globalShim->functionList); funcIdx++) - { - const TestDefShimFunction *const globalFunction = lstGet(globalShim->functionList, funcIdx); - const TestDefShimFunction function = - { - .name = strDup(globalFunction->name), - .inc = strDup(globalFunction->inc), - }; - - lstAdd(functionList, &function); - } - - const TestDefShim testDefShim = - { - .name = strDup(globalShim->name), - .integration = globalShim->integration, - .functionList = functionList, - }; - - lstAdd(shimList, &testDefShim); - } - } - MEM_CONTEXT_OBJ_END(); - - testDefModule.shimList = shimList; - } - MEM_CONTEXT_OBJ_END(); - - lstAdd(moduleList, &testDefModule); - - // Add feature to global list - if (testDefModule.feature != NULL) - strLstAdd(globalFeatureList, testDefModule.feature); - } - YAML_SEQ_END(); - } - YAML_MAP_END(); - } - YAML_SEQ_END(); - } - - FUNCTION_LOG_RETURN_VOID(); -} - -/**********************************************************************************************************************************/ -TestDef -testDefParse(const Storage *const storageRepo) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STORAGE, storageRepo); - FUNCTION_LOG_END(); - - FUNCTION_AUDIT_STRUCT(); - - // Module list - List *const moduleList = lstNewP(sizeof(TestDefModule), .comparator = lstComparatorStr); - - MEM_CONTEXT_TEMP_BEGIN() - { - // Initialize yaml - Yaml *const yaml = yamlNew(storageGetP(storageNewReadP(storageRepo, STRDEF("test/define.yaml")))); - yamlEventNextCheck(yaml, yamlEventTypeMapBegin); - - // Parse unit tests - testDefParseModuleList(yaml, moduleList); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN_STRUCT((TestDef){.moduleList = moduleList}); -} diff --git a/test/src/command/test/define.h b/test/src/command/test/define.h deleted file mode 100644 index d5e9c3ab4..000000000 --- a/test/src/command/test/define.h +++ /dev/null @@ -1,79 +0,0 @@ -/*********************************************************************************************************************************** -Parse Define Yaml -***********************************************************************************************************************************/ -#ifndef TEST_COMMAND_TEST_DEFINE_H -#define TEST_COMMAND_TEST_DEFINE_H - -#include "build/common/string.h" -#include "common/type/list.h" -#include "storage/storage.h" - -// Test definition types -typedef enum -{ - testDefTypeUnit, - testDefTypeIntegration, - testDefTypePerformance, -} TestDefType; - -// Covered code modules -typedef struct TestDefCoverage -{ - const String *name; // Code module name - bool coverable; // Does this code module include coverable code? - bool included; // Is this module included in another module? -} TestDefCoverage; - -// Harness modules -typedef struct TestDefHarness -{ - const String *name; // Harness module name - bool integration; // Include in integration tests? - const StringList *includeList; // List of modules to include directly in harness -} TestDefHarness; - -// Shimmed function -typedef struct TestDefShimFunction -{ - const String *name; // Function name - const String *inc; // Included module the function is defined in (NULL if in shim) -} TestDefShimFunction; - -// Shimmed code modules -typedef struct TestDefShim -{ - const String *name; // Shim module name - bool integration; // Include in integration tests? - const List *functionList; // List of TestDefShimFunction to shim -} TestDefShim; - -typedef struct TestDefModule -{ - const String *name; // Test module name - TestDefType type; // Module type (unit, performance, etc.) - unsigned int total; // Total sub-tests - bool pgRequired; // Is PostgreSQL required? - bool binRequired; // Is a binary required to run this test? - bool containerRequired; // Is a container required to run this test? - const String *flag; // Compilation flags - const String *feature; // Does this module introduce a feature? - const StringList *featureList; // Features to include in this module - const List *coverageList; // Code modules covered by this test module - const StringList *dependList; // Code modules that this test module depends on - const StringList *includeList; // Additional code modules to include in the test module - const List *harnessList; // Harnesses to use for this test - const List *shimList; // Shims for this test -} TestDefModule; - -typedef struct TestDef -{ - const List *moduleList; // Module list -} TestDef; - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Parse define.yaml -TestDef testDefParse(const Storage *const storageRepo); - -#endif diff --git a/test/src/command/test/lint.c b/test/src/command/test/lint.c deleted file mode 100644 index c637a107d..000000000 --- a/test/src/command/test/lint.c +++ /dev/null @@ -1,262 +0,0 @@ -/*********************************************************************************************************************************** -Code Linter -***********************************************************************************************************************************/ -#include - -#include - -#include "build/common/regExp.h" -#include "build/common/render.h" -#include "command/test/lint.h" -#include "common/log.h" -#include "storage/posix/storage.h" -#include "storage/storage.h" - -/*********************************************************************************************************************************** -Validate StringId values -***********************************************************************************************************************************/ -static unsigned int -lintStrId(const String *const source) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STRING, source); - FUNCTION_LOG_END(); - - unsigned int result = 0; - - MEM_CONTEXT_TEMP_BEGIN() - { - // Check all StringIds - RegExp *const strIdExp = regExpNew(STRDEF("STRID(5|6)(S){0,1}\\([^)]+\\)")); - const char *sourcePtr = strZ(source); - - while (regExpMatch(strIdExp, STRDEF(sourcePtr))) - { - const String *const match = regExpMatchStr(strIdExp, STRDEF(sourcePtr)); - const StringList *const paramList = strLstNewSplitZ( - strLstGet(strLstNewSplitZ(strSubN(match, 0, strSize(match) - 1), "("), 1), ","); - const String *const param = strTrim(strLstGet(paramList, 0)); - - sourcePtr = regExpMatchPtr(strIdExp, STRDEF(sourcePtr)) + strSize(match); - - // Skip macro definitions - if (strEqZ(match, "STRID5(str, strId)") || strEqZ(match, "STRID5S(str, seq, strId)") || - strEqZ(match, "STRID6(str, strId)") || strEqZ(match, "STRID6S(str, seq, strId)")) - continue; - - // Skip test strings - if (strBeginsWithZ(param, "\\\"") && strEndsWithZ(param, "\\\"")) - continue; - - // Skip test values - if (strLstSize(paramList) > 1 && strBeginsWithZ(strTrim(strLstGet(paramList, 1)), "TEST_")) - continue; - - // Param must begin and end with a quote - if (strZ(param)[0] != '"' || strZ(param)[strSize(param) - 1] != '"') - { - LOG_WARN_FMT("'%s' must have quotes around string parameter '%s'", strZ(match), strZ(param)); - result++; - - continue; - } - - // Check validity of the string - const String *expected; - - if (strBeginsWithZ(match, "STRID5" "(") || strBeginsWithZ(match, "STRID6" "(")) - expected = bldStrId(strZ(strSubN(param, 1, strSize(param) - 2))); - else - { - expected = bldStrIdSeq( - strZ(strSubN(param, 1, strSize(param) - 2)), cvtZToUInt(strZ(strTrim(strLstGet(paramList, 1))))); - } - - if (!strEq(match, expected)) - { - LOG_WARN_FMT("'%s' should be '%s'", strZ(match), strZ(expected)); - result++; - } - } - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN(UINT, result); -} - -/*********************************************************************************************************************************** -Detect characters that could be used to hide code, e.g. invisible, zero-width, bidirectional, or homoglyph characters. Source must -be 7-bit ASCII with tab and newline as the only permitted control characters. Intentional non-ASCII byte values must be written as -\xNN escapes so they remain visible in the source. -***********************************************************************************************************************************/ -static unsigned int -lintChar(const String *const source) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STRING, source); - FUNCTION_LOG_END(); - - unsigned int result = 0; - - const unsigned char *const data = (const unsigned char *)strZ(source); - const size_t size = strSize(source); - unsigned int line = 1; - - for (size_t idx = 0; idx < size; idx++) - { - const unsigned char chr = data[idx]; - - // Count lines for reporting - if (chr == '\n') - { - line++; - continue; - } - - // Tab is the only other permitted control character - if (chr == '\t') - continue; - - // Flag any byte outside printable 7-bit ASCII. Invisible, bidirectional, and homoglyph characters all live here, so they - // are rejected wholesale and intentional non-ASCII must be written with \xNN escapes. - if (chr < 0x20 || chr > 0x7E) - { - // Decode the code point for reporting on a best-effort basis. Validation is unnecessary since the character is rejected - // regardless, so the byte count is taken from the lead byte and limited to the bytes available. - unsigned int seqSize = 1; - - if (chr >= 0xF0) - seqSize = 4; - else if (chr >= 0xE0) - seqSize = 3; - else if (chr >= 0xC0) - seqSize = 2; - - if (seqSize > size - idx) - seqSize = (unsigned int)(size - idx); - - unsigned int codePoint = chr; - - if (seqSize > 1) - { - codePoint = chr & (0xFF >> (seqSize + 1)); - - for (unsigned int seqIdx = 1; seqIdx < seqSize; seqIdx++) - codePoint = (codePoint << 6) | (data[idx + seqIdx] & 0x3F); - } - - LOG_WARN_FMT("line %u contains disallowed character U+%04X (source must be 7-bit ASCII)", line, codePoint); - result++; - - idx += seqSize - 1; - } - } - - FUNCTION_LOG_RETURN(UINT, result); -} - -/*********************************************************************************************************************************** -Files that are exempt from the content checks. Binary files are unscannable places to hide content, so each entry must be a -deliberate, reviewable decision. -***********************************************************************************************************************************/ -static const char *const lintFileSkipList[] = -{ - "doc/resource/logo.png", // Project logo - "test/data/filecopy.table.bin", // Binary test fixture - "doc/resource/git-history.cache", // Generated from git history, contains non-ASCII author names -}; - -static bool -lintFileSkip(const String *const name) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(STRING, name); - FUNCTION_TEST_END(); - - bool result = false; - - for (unsigned int skipIdx = 0; skipIdx < LENGTH_OF(lintFileSkipList); skipIdx++) - { - if (strEqZ(name, lintFileSkipList[skipIdx])) - { - result = true; - break; - } - } - - FUNCTION_TEST_RETURN(BOOL, result); -} - -/**********************************************************************************************************************************/ -void -lintAll(const String *const pathRepo) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STRING, pathRepo); - FUNCTION_LOG_END(); - - ASSERT(pathRepo != NULL); - - MEM_CONTEXT_TEMP_BEGIN() - { - const Storage *const storageRepo = storagePosixNewP(pathRepo); - StorageIterator *const repoItr = storageNewItrP( - storageRepo, NULL, .level = storageInfoLevelType, .recurse = true, .sortOrder = sortOrderAsc); - - while (storageItrMore(repoItr)) - { - const StorageInfo info = storageItrNext(repoItr); - - // Skip anything that is not a file and files that are exempt from the content checks - if (info.type != storageTypeFile || lintFileSkip(info.name)) - continue; - - // Read the file, skipping any that cannot be read. Committed files are always readable after checkout, so this only - // occurs from local filesystem state, and readability is enforced by the build rather than the linter. - const Buffer *content = NULL; - - TRY_BEGIN() - { - content = storageGetP(storageNewReadP(storageRepo, info.name)); - } - CATCH(FileOpenError) - { - // File cannot be read, so there is nothing to check - } - TRY_END(); - - if (content == NULL) - continue; - - unsigned int errorTotal = 0; - - // A binary file cannot be scanned for hidden content, so it is not allowed unless added to lintFileSkipList - if (memchr(bufPtrConst(content), '\0', bufUsed(content)) != NULL) - { - LOG_WARN("unexpected binary file (add to the linter skip list if intentional)"); - errorTotal = 1; - } - // Otherwise the file must be 7-bit ASCII text, with the StringId check applied to C source - else - { - const String *const source = strNewBuf(content); - - errorTotal = lintChar(source); - - // The StringId check applies to C source, including hand-written .c.inc, but not generated or vendored includes - if (strEndsWithZ(info.name, ".c") || strEndsWithZ(info.name, ".h") || - (strEndsWithZ(info.name, ".c.inc") && !strEndsWithZ(info.name, ".auto.c.inc") && - !strEndsWithZ(info.name, ".vendor.c.inc"))) - { - errorTotal += lintStrId(source); - } - } - - if (errorTotal > 0) - THROW_FMT(FormatError, "%u linter error(s) in '%s' (see warnings above)", errorTotal, strZ(info.name)); - } - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN_VOID(); -} diff --git a/test/src/command/test/lint.h b/test/src/command/test/lint.h deleted file mode 100644 index 91424755d..000000000 --- a/test/src/command/test/lint.h +++ /dev/null @@ -1,14 +0,0 @@ -/*********************************************************************************************************************************** -Test Build Handler -***********************************************************************************************************************************/ -#ifndef TEST_COMMAND_TEST_LINT_H -#define TEST_COMMAND_TEST_LINT_H - -#include "common/type/string.h" - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -void lintAll(const String *pathRepo); - -#endif diff --git a/test/src/command/test/test.c b/test/src/command/test/test.c deleted file mode 100644 index 29003e44b..000000000 --- a/test/src/command/test/test.c +++ /dev/null @@ -1,206 +0,0 @@ -/*********************************************************************************************************************************** -Test Command -***********************************************************************************************************************************/ -#include - -#include - -#include "build/common/exec.h" -#include "command/test/build.h" -#include "command/test/lint.h" -#include "command/test/test.h" -#include "common/debug.h" -#include "common/log.h" -#include "config/config.h" -#include "storage/posix/storage.h" - -/**********************************************************************************************************************************/ -static void -cmdTestPathCreate(const Storage *const storage, const String *const path) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STORAGE, storage); - FUNCTION_LOG_PARAM(STRING, path); - FUNCTION_LOG_END(); - - FUNCTION_AUDIT_HELPER(); - - ASSERT(storage != NULL); - - const String *const rmCommand = strNewFmt("rm -rf '%s'/*", strZ(storagePathP(storage, path))); - - TRY_BEGIN() - { - execOneExpectP(rmCommand); - } - CATCH_ANY() - { - // Reset permissions - execOneExpectP(strNewFmt("chmod -R 777 '%s'", strZ(storagePathP(storage, path)))); - - // Try to remove again - execOneExpectP(rmCommand); - } - TRY_END(); - - storagePathCreateP(storage, path, .mode = 0770); - - FUNCTION_LOG_RETURN_VOID(); -} - -/*********************************************************************************************************************************** -Fix vm architectures to project standard -***********************************************************************************************************************************/ -static const String * -cmdTestVmArchFix(const String *vmArch) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STRING, vmArch); - FUNCTION_LOG_END(); - - if (strEqZ(vmArch, "i686")) - vmArch = strNewZ("i386"); - else if (strEqZ(vmArch, "arm64")) - vmArch = strNewZ("aarch64"); - - FUNCTION_LOG_RETURN_CONST(STRING, vmArch); -} - -/**********************************************************************************************************************************/ -void -cmdTest( - const String *const pathRepo, const String *const pathTest, const String *vm, const unsigned int vmId, - const String *const pgVersion, const String *moduleName, const unsigned int test, const uint64_t scale, const LogLevel logLevel, - const bool logTime, const String *const timeZone, const String *architecture, const bool coverage, const bool profile, - const bool optimize, const bool backTrace) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STRING, pathRepo); - FUNCTION_LOG_PARAM(STRING, pathTest); - FUNCTION_LOG_PARAM(STRING, vm); - FUNCTION_LOG_PARAM(UINT, vmId); - FUNCTION_LOG_PARAM(STRING, pgVersion); - FUNCTION_LOG_PARAM(STRING, moduleName); - FUNCTION_LOG_PARAM(UINT, test); - FUNCTION_LOG_PARAM(UINT64, scale); - FUNCTION_LOG_PARAM(ENUM, logLevel); - FUNCTION_LOG_PARAM(BOOL, logTime); - FUNCTION_LOG_PARAM(STRING, timeZone); - FUNCTION_LOG_PARAM(STRING, architecture); - FUNCTION_LOG_PARAM(BOOL, coverage); - FUNCTION_LOG_PARAM(BOOL, profile); - FUNCTION_LOG_PARAM(BOOL, optimize); - FUNCTION_LOG_PARAM(BOOL, backTrace); - FUNCTION_LOG_END(); - - MEM_CONTEXT_TEMP_BEGIN() - { - // Linter - lintAll(pathRepo); - - // Find test - ASSERT(moduleName != NULL); - - const TestDef testDef = testDefParse(storagePosixNewP(pathRepo)); - const TestDefModule *const module = lstFind(testDef.moduleList, &moduleName); - - CHECK_FMT(ParamInvalidError, module != NULL, "'%s' is not a valid test", strZ(moduleName)); - - // Vm used for integration and the pgbackrest binary - const String *const vmInt = vm; - - // If integration then the vm should be none. Integration tests do not run in containers but instead spawn their own. - if (module->type == testDefTypeIntegration) - vm = strNewZ("none"); - - // Get test architecture - if (architecture == NULL) - architecture = cmdTestVmArchFix(strTrim(execOneExpectP(STRDEF("uname -m")))); - - // Build test - bool buildRetry = false; - const String *const pathUnit = strNewFmt("%s/unit-%u/%s", strZ(pathTest), vmId, strZ(vm)); - const String *const pathUnitBuild = strNewFmt("%s/build", strZ(pathUnit)); - const Storage *const storageUnitBuild = storagePosixNewP(pathUnitBuild, .write = true); - - do - { - TRY_BEGIN() - { - // Build unit - TestBuild *const testBld = testBldNew( - pathRepo, pathTest, vm, vmInt, vmId, pgVersion, module, test, scale, logLevel, logTime, timeZone, architecture, - coverage, profile, optimize, backTrace); - testBldUnit(testBld); - - // Meson setup - String *const mesonSetup = strCatZ(strNew(), "-Dbuildtype="); - - if (module->flag != NULL || profile || module->type == testDefTypePerformance) - { - ASSERT(module->flag == NULL || strEqZ(module->flag, "-DNDEBUG")); - strCatZ(mesonSetup, "release"); - } - else - strCatZ(mesonSetup, "debug"); - - strCatFmt(mesonSetup, " -Db_coverage=%s", cvtBoolToConstZ(coverage)); - - if (!storageExistsP(testBldStorageTest(testBld), strNewFmt("%s/build.ninja", strZ(pathUnitBuild)))) - { - LOG_DETAIL("meson setup"); - - execOneExpectP( - strNewFmt( - "meson setup -Dwerror=true -Dfatal-errors=true %s '%s' '%s'", strZ(mesonSetup), strZ(pathUnitBuild), - strZ(pathUnit))); - } - // Else reconfigure - else - { - LOG_DETAIL("meson configure"); - - execOneExpectP(strNewFmt("meson configure %s '%s'", strZ(mesonSetup), strZ(pathUnitBuild))); - } - - // Remove old coverage data - const String *const pathCoverage = STRDEF("test-unit.p"); - - StorageIterator *const storageItr = storageNewItrP( - storageUnitBuild, pathCoverage, .expression = STRDEF("\\.gcda$")); - - while (storageItrMore(storageItr)) - { - storageRemoveP( - storageUnitBuild, strNewFmt("%s/%s", strZ(pathCoverage), strZ(storageItrNext(storageItr).name))); - } - - // Remove old profile data - storageRemoveP(storageUnitBuild, STRDEF("gmon.out")); - - // Ninja build - execOneExpectP(strNewFmt("ninja -C '%s'", strZ(pathUnitBuild))); - buildRetry = false; - } - CATCH_ANY() - { - // If this is the first build failure then clean the build path and retry - if (buildRetry == false) - { - buildRetry = true; - - LOG_WARN_FMT("build failed for unit %s -- will retry: %s", strZ(moduleName), errorMessage()); - cmdTestPathCreate(storagePosixNewP(pathTest, .write = true), pathUnit); - } - // Else error - else - THROWP_FMT(errorType(), "build failed for unit %s: %s", strZ(moduleName), errorMessage()); - } - TRY_END(); - } - while (buildRetry); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN_VOID(); -} diff --git a/test/src/command/test/test.h b/test/src/command/test/test.h deleted file mode 100644 index ada7d0e4b..000000000 --- a/test/src/command/test/test.h +++ /dev/null @@ -1,20 +0,0 @@ -/*********************************************************************************************************************************** -Test Command - -Perform a test. -***********************************************************************************************************************************/ -#ifndef TEST_COMMAND_TEST_TEST_H -#define TEST_COMMAND_TEST_TEST_H - -#include "common/logLevel.h" -#include "common/type/stringList.h" - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -void cmdTest( - const String *pathRepo, const String *pathTest, const String *const vm, unsigned int vmId, const String *pgVersion, - const String *moduleName, unsigned int test, uint64_t scale, LogLevel logLevel, bool logTime, const String *timeZone, - const String *architecture, bool coverage, bool profile, bool optimize, bool backTrace); - -#endif diff --git a/test/src/config/load.c b/test/src/config/load.c deleted file mode 100644 index fdbedd095..000000000 --- a/test/src/config/load.c +++ /dev/null @@ -1,131 +0,0 @@ -/*********************************************************************************************************************************** -Configuration Load -***********************************************************************************************************************************/ -#include - -#include - -#include "command/command.h" -#include "common/debug.h" -#include "common/io/io.h" -#include "common/log.h" -#include "config/config.intern.h" -#include "config/load.h" -#include "storage/posix/storage.h" - -/*********************************************************************************************************************************** -Load log settings -***********************************************************************************************************************************/ -static void -cfgLoadLogSetting(void) -{ - FUNCTION_LOG_VOID(logLevelTrace); - - // Initialize logging - LogLevel logLevelConsole = logLevelOff; - bool logTimestamp = true; - - if (cfgOptionValid(cfgOptLogLevel)) - logLevelConsole = logLevelEnum(cfgOptionSeq(cfgOptLogLevel)); - - if (cfgOptionValid(cfgOptLogTimestamp)) - logTimestamp = cfgOptionBool(cfgOptLogTimestamp); - - logInit(logLevelConsole, logLevelOff, logLevelOff, logTimestamp, 0, 1, false); - - FUNCTION_LOG_RETURN_VOID(); -} - -/*********************************************************************************************************************************** -Update options that have complex rules -***********************************************************************************************************************************/ -static void -cfgLoadUpdateOption(void) -{ - FUNCTION_LOG_VOID(logLevelTrace); - - // Get current working dir - char currentWorkDir[1024]; - THROW_ON_SYS_ERROR(getcwd(currentWorkDir, sizeof(currentWorkDir)) == NULL, FormatError, "unable to get cwd"); - - // If repo-path is relative then make it absolute - if (cfgOptionValid(cfgOptRepoPath)) - { - const String *const repoPath = cfgOptionStr(cfgOptRepoPath); - - if (!strBeginsWithZ(repoPath, "/")) - { - cfgOptionSet( - cfgOptRepoPath, cfgOptionSource(cfgOptRepoPath), VARSTR(strNewFmt("%s/%s", currentWorkDir, strZ(repoPath)))); - } - } - - // If test-path is relative then make it absolute - if (cfgOptionValid(cfgOptTestPath)) - { - const String *const testPath = cfgOptionStr(cfgOptTestPath); - - if (!strBeginsWithZ(testPath, "/")) - { - cfgOptionSet( - cfgOptTestPath, cfgOptionSource(cfgOptTestPath), VARSTR(strNewFmt("%s/%s", currentWorkDir, strZ(testPath)))); - } - } - - FUNCTION_LOG_RETURN_VOID(); -} - -/**********************************************************************************************************************************/ -void -cfgLoad(unsigned int argListSize, const char *argList[]) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(UINT, argListSize); - FUNCTION_LOG_PARAM(CHARPY, argList); - FUNCTION_LOG_END(); - - MEM_CONTEXT_TEMP_BEGIN() - { - // Make a copy of the arguments so they can be manipulated - StringList *const argListNew = strLstNew(); - - for (unsigned int argListIdx = 0; argListIdx < argListSize; argListIdx++) - strLstAddZ(argListNew, argList[argListIdx]); - - // Parse config from command line - TRY_BEGIN() - { - cfgParseP(storagePosixNewP(FSLASH_STR), strLstSize(argListNew), strLstPtr(argListNew), .noConfigLoad = true); - } - CATCH(CommandRequiredError) - { - strLstAddZ(argListNew, CFGCMD_TEST); - cfgParseP(storagePosixNewP(FSLASH_STR), strLstSize(argListNew), strLstPtr(argListNew), .noConfigLoad = true); - } - TRY_END(); - - // Error on noop command. This command is required to hold options that must be declared but are unused by test. - if (cfgCommand() == cfgCmdNoop) - THROW(CommandInvalidError, "invalid command '" CFGCMD_NOOP "'"); - - // Load the log settings - cfgLoadLogSetting(); - - // Neutralize the umask to make the repository file/path modes more consistent - if (cfgOptionValid(cfgOptNeutralUmask) && cfgOptionBool(cfgOptNeutralUmask)) - umask(0000); - - // Set IO buffer size - if (cfgOptionValid(cfgOptBufferSize)) - ioBufferSizeSet(cfgOptionUInt(cfgOptBufferSize)); - - // Update options that have complex rules - cfgLoadUpdateOption(); - - // Begin the command - cmdBegin(); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN_VOID(); -} diff --git a/test/src/config/load.h b/test/src/config/load.h deleted file mode 100644 index 475ad9759..000000000 --- a/test/src/config/load.h +++ /dev/null @@ -1,13 +0,0 @@ -/*********************************************************************************************************************************** -Test Configuration Load -***********************************************************************************************************************************/ -#ifndef TEST_CONFIG_LOAD_H -#define TEST_CONFIG_LOAD_H - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Load the configuration -void cfgLoad(unsigned int argListSize, const char *argList[]); - -#endif diff --git a/test/src/config/meson.build b/test/src/config/meson.build deleted file mode 100644 index 71dcdc7e2..000000000 --- a/test/src/config/meson.build +++ /dev/null @@ -1,16 +0,0 @@ -#################################################################################################################################### -# Generate config -#################################################################################################################################### -test_parse_auto_c_inc = custom_target( - 'test_parse.auto.c.inc', - output: 'parse.auto.c.inc', - depend_files: [ - '../build/config/config.yaml', - ], - command: [ - build_code, - 'config', - '@CURRENT_SOURCE_DIR@/../..', - '@BUILD_ROOT@/test', - ], -) diff --git a/test/src/main.c b/test/src/main.c deleted file mode 100644 index 0a64d3a6b..000000000 --- a/test/src/main.c +++ /dev/null @@ -1,111 +0,0 @@ -/*********************************************************************************************************************************** -Main -***********************************************************************************************************************************/ -#include - -#include - -#include "command/command.h" -#include "command/exit.h" -#include "command/help/help.h" -#include "command/test/coverage.h" -#include "command/test/test.h" -#include "common/debug.h" -#include "common/log.h" -#include "common/macro.h" -#include "common/memContext.h" -#include "common/stat.h" -#include "config/load.h" -#include "config/parse.h" -#include "storage/posix/storage.h" -#include "version.h" - -/*********************************************************************************************************************************** -Include automatically generated help data -***********************************************************************************************************************************/ -#include "command/help/help.auto.c.inc" - -int -main(int argListSize, const char *argList[]) -{ - // Set stack trace and mem context error cleanup handlers - static const ErrorHandlerFunction errorHandlerList[] = {stackTraceClean, memContextClean}; - errorHandlerSet(errorHandlerList, LENGTH_OF(errorHandlerList)); - - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(INT, argListSize); - FUNCTION_LOG_PARAM(CHARPY, argList); - FUNCTION_LOG_END(); - - // Initialize command with the start time - cmdInit(); - - // Initialize statistics collector - statInit(); - - // Initialize exit handler - exitInit(); - - // Process commands - volatile int result = 0; - volatile bool error = false; - - TRY_BEGIN() - { - // Load the configuration - // ------------------------------------------------------------------------------------------------------------------------- - cfgLoad((unsigned int)argListSize, argList); - - // Handle command - // ------------------------------------------------------------------------------------------------------------------------- - switch (cfgCommandHelp() ? cfgCmdHelp : cfgCommand()) - { - // Test - // ----------------------------------------------------------------------------------------------------------------- - case cfgCmdTest: - { - // Run a single test - if (cfgOptionTest(cfgOptVmId)) - { - cmdTest( - cfgOptionStr(cfgOptRepoPath), cfgOptionStr(cfgOptTestPath), cfgOptionStr(cfgOptVm), - cfgOptionUInt(cfgOptVmId), cfgOptionStr(cfgOptPgVersion), strLstGet(cfgCommandParam(), 0), - cfgOptionTest(cfgOptTest) ? cfgOptionUInt(cfgOptTest) : 0, cfgOptionUInt64(cfgOptScale), - logLevelEnum(cfgOptionSeq(cfgOptLogLevelTest)), cfgOptionBool(cfgOptLogTimestamp), - cfgOptionStrNull(cfgOptTz), cfgOptionStrNull(cfgOptVmArch), cfgOptionBool(cfgOptCoverage), - cfgOptionBool(cfgOptProfile), cfgOptionBool(cfgOptOptimize), cfgOptionBool(cfgOptBackTrace)); - } - // Top-level test - else - { - result = testCvgGenerate( - cfgOptionStr(cfgOptRepoPath), cfgOptionStr(cfgOptTestPath), cfgOptionStr(cfgOptVm), - cfgOptionBool(cfgOptCoverageSummary), cfgCommandParam()) > 0; - } - - break; - } - - // Help/version - // ----------------------------------------------------------------------------------------------------------------- - case cfgCmdHelp: - case cfgCmdVersion: - cmdHelp(BUF(helpData, sizeof(helpData))); - break; - - // Error on commands that should have been handled - // ----------------------------------------------------------------------------------------------------------------- - case cfgCmdNoop: - THROW_FMT(AssertError, "'%s' command should have been handled", cfgCommandName()); - break; - } - } - CATCH_FATAL() - { - error = true; - result = exitSafe(result, true, 0); - } - TRY_END(); - - FUNCTION_LOG_RETURN(INT, error ? result : exitSafe(result, false, 0)); -} diff --git a/test/src/meson.build b/test/src/meson.build deleted file mode 100644 index 646612402..000000000 --- a/test/src/meson.build +++ /dev/null @@ -1,72 +0,0 @@ -#################################################################################################################################### -# Write configuration -#################################################################################################################################### -configure_file(output: 'build.auto.h', configuration: configuration) - -#################################################################################################################################### -# Build config target -#################################################################################################################################### -# build parse.auto.c.inc -subdir('config') - -#################################################################################################################################### -# Build help target -#################################################################################################################################### -# build help.auto.c.inc -subdir('command/help') - -#################################################################################################################################### -# test target -#################################################################################################################################### -src_test = [ - '../../src/build/common/exec.c', - '../../src/build/common/json.c', - '../../src/build/common/regExp.c', - '../../src/build/common/render.c', - '../../src/build/common/string.c', - '../../src/build/common/yaml.c', - '../../src/command/command.c', - '../../src/command/exit.c', - '../../src/command/help/help.c', - '../../src/command/lock.c', - '../../src/common/compress/common.c', - '../../src/common/compress/bz2/common.c', - '../../src/common/compress/bz2/decompress.c', - '../../src/common/fork.c', - '../../src/common/ini.c', - '../../src/common/io/fd.c', - '../../src/common/io/fdRead.c', - '../../src/common/io/fdWrite.c', - '../../src/common/lock.c', - '../../src/common/stat.c', - '../../src/config/config.c', - '../../src/config/parse.c', - 'command/test/build.c', - 'command/test/coverage.c', - 'command/test/define.c', - 'command/test/lint.c', - 'command/test/test.c', - 'config/load.c', - 'main.c', -] - -executable( - 'test-pgbackrest', - src_common, - src_test, - test_help_auto_c_inc, - test_parse_auto_c_inc, - include_directories: include_directories('.', '../../src'), - c_args: [ - # Use large buffer sizes to capture large error/log outputs - '-DERROR_MESSAGE_BUFFER_SIZE=262144', - '-DLOG_BUFFER_SIZE=262144', - arg_unity, - ], - dependencies: [ - lib_backtrace, - lib_bz2, - lib_yaml, - ], - build_by_default: false, -) diff --git a/test/src/module/test/command/coverage/coverage_test.py b/test/src/module/test/command/coverage/coverage_test.py new file mode 100644 index 000000000..5cf48ab9d --- /dev/null +++ b/test/src/module/test/command/coverage/coverage_test.py @@ -0,0 +1,492 @@ +"""Test Coverage Testing and Reporting.""" + +#################################################################################################################################### +import io +import json +import os +import tempfile +from contextlib import redirect_stdout + +from harness.test import * + +from command.coverage.coverage import * +from command.coverage.coverage import Coverage, CoverageFile, CoverageFunction, CoverageLine +from command.test.define import TEST_LANG_C, TEST_LANG_PYTHON +from common.error import * +from common.log import * + +# A define file with one C test and one python test, which is what the coverage command reads to know what should be covered +DEFINE = """ +unit: + - name: common + + test: + - name: error + + coverage: + - common/error + - common/errorInternal: noCode + + - name: stack-trace + + coverage: + - common/stackTrace + + - name: test + lang: python + + test: + - name: common/log + + coverage: + - test/common/log + +integration: [] +performance: [] +""" + +# Source of the C module, which the report is rendered against +SOURCE_C = "/***/\nvoid\nfuncOne(void)\n{\n doOne();\n}\n" + +# Source of the python module +SOURCE_PY = "def func(a):\n if a:\n return 1\n\n return 0\n" + + +#################################################################################################################################### +class Config: + """What the coverage command reads from the command line.""" + + def __init__(self, repo_path, test_path, module, vm="none", coverage_summary=False): + self.repo_path = repo_path + self.test_path = test_path + self.module = module + self.vm = vm + self.coverage_summary = coverage_summary + + +#################################################################################################################################### +def _line(no, hit, branch_list=None, target_list=None): + """Build a coverage line.""" + + return CoverageLine(no, hit, branch_list, target_list) + + +#################################################################################################################################### +def _file(name, line_list, lang=TEST_LANG_C, function_list=()): + """Build a coverage file from the lines and functions given.""" + + result = CoverageFile(name, lang) + result.line_list = list(line_list) + result.function_list = list(function_list) + + return result + + +#################################################################################################################################### +def _coverage(file_list): + """Build a coverage object from the files given.""" + + result = Coverage() + result.file_list = list(file_list) + + return result + + +#################################################################################################################################### +def test_coverage_file_find(): + """A file is found by code module name and missing is reported by returning nothing.""" + + coverage = _coverage([_file("src/common/error.c", []), _file("src/common/log.c", [])]) + + assert_equal(coverage.file_find("src/common/log.c").name, "src/common/log.c") + assert_is_none(coverage.file_find("src/common/bogus.c")) + + +#################################################################################################################################### +def test_coverage_total(): + """Totals are counted over the lines and functions, and a file is covered only when every line and branch was.""" + + file = _file( + "src/common/error.c", + [_line(5, 3), _line(7, 0), _line(9, 1, [3, 0])], + function_list=[CoverageFunction("errorNew", 1, 10, 3), CoverageFunction("errorFree", 12, 20, 0)], + ) + coverage = _coverage([file]) + + coverage.total_calculate() + + assert_equal((file.line_hit, file.line_total), (2, 3)) + assert_equal((file.branch_hit, file.branch_total), (1, 2)) + assert_equal((file.function_hit, file.function_total), (1, 2)) + assert_false(file.covered()) + + # A file with everything hit is covered, and so is one with nothing in it at all + file = _file("src/common/log.c", [_line(5, 1), _line(7, 2, [1, 1])]) + coverage = _coverage([file, _file("src/common/empty.c", [])]) + + coverage.total_calculate() + + assert_true(file.covered()) + assert_true(coverage.file_list[1].covered()) + + +#################################################################################################################################### +def test_coverage_covered_remove(): + """Only what is missing is left, since the report is about what still needs a test.""" + + file = _file( + "src/common/error.c", + [_line(5, 3), _line(7, 0), _line(9, 1, [3, 0]), _line(11, 1, [1, 1]), _line(13, 0, [0, 0])], + ) + coverage = _coverage([file]) + + coverage.covered_remove() + + # A line that ran with every branch taken is gone; a line that never ran or has a branch that was never taken stays + assert_equal([line.no for line in file.line_list], [7, 9, 13]) + + +#################################################################################################################################### +def test_coverage_merge_c(): + """C coverage is summed line by line and branch by branch, since both runs compiled the same source.""" + + coverage = _coverage( + [ + _file( + "src/common/error.c", + [_line(5, 1), _line(7, 0, [1, 0])], + function_list=[CoverageFunction("errorNew", 1, 10, 1)], + ) + ] + ) + + coverage.merge( + _coverage( + [ + _file( + "src/common/error.c", + [_line(5, 2), _line(7, 3, [0, 4])], + function_list=[CoverageFunction("errorNew", 1, 10, 2)], + ), + # A file the first run did not report is taken as it is + _file("src/common/log.c", [_line(3, 1)]), + ] + ) + ) + + file = coverage.file_find("src/common/error.c") + + assert_equal([(line.no, line.hit) for line in file.line_list], [(5, 3), (7, 3)]) + assert_equal(file.line_list[1].branch_list, [1, 4]) + assert_equal(file.function_list[0].hit, 3) + + # Files are sorted so the report is in a fixed order however the runs came back + assert_equal([file.name for file in coverage.file_list], ["src/common/error.c", "src/common/log.c"]) + + +#################################################################################################################################### +def test_coverage_merge_c_error(): + """C coverage that does not line up means the runs were built from different source, which cannot be merged.""" + + def merge(line_list, function_list=(), merge_line_list=None, merge_function_list=()): + coverage = _coverage([_file("src/common/error.c", line_list, function_list=function_list)]) + + coverage.merge( + _coverage([_file("src/common/error.c", merge_line_list, function_list=merge_function_list)]), + ) + + # A different number of lines + with assert_raises(TestError) as error: + merge([_line(5, 1)], merge_line_list=[_line(5, 1), _line(7, 1)]) + + assert_equal(str(error.exception), "coverage for 'src/common/error.c' does not match the prior run") + + # The same number of lines but not the same lines + with assert_raises(TestError) as error: + merge([_line(5, 1)], merge_line_list=[_line(7, 1)]) + + assert_equal(str(error.exception), "coverage line mismatch in 'src/common/error.c'") + + # A line that has branches in one run and not the other + with assert_raises(TestError) as error: + merge([_line(5, 1, [1, 0])], merge_line_list=[_line(5, 1)]) + + assert_equal(str(error.exception), "coverage branches for 'src/common/error.c' do not match the prior run") + + # A function that is not in the other run + with assert_raises(TestError) as error: + merge( + [_line(5, 1)], + function_list=[CoverageFunction("errorNew", 1, 10, 1)], + merge_line_list=[_line(5, 1)], + merge_function_list=[CoverageFunction("errorFree", 1, 10, 1)], + ) + + assert_equal(str(error.exception), "coverage for function 'errorNew' is missing") + + # A function that does not span the same lines + with assert_raises(TestError) as error: + merge( + [_line(5, 1)], + function_list=[CoverageFunction("errorNew", 1, 10, 1)], + merge_line_list=[_line(5, 1)], + merge_function_list=[CoverageFunction("errorNew", 1, 12, 1)], + ) + + assert_equal(str(error.exception), "coverage for function 'errorNew' does not match the prior run") + + +#################################################################################################################################### +def test_coverage_merge_python(): + """Python coverage is merged by line and by where an arc goes, since a run reports only what it knows about.""" + + coverage = _coverage( + [ + _file( + "test/lib/common/log.py", + [_line(2, 1, [1, 0], [3, 5]), _line(5, 0)], + TEST_LANG_PYTHON, + ) + ] + ) + + coverage.merge( + _coverage( + [ + _file( + "test/lib/common/log.py", + [ + # An arc the first run reported, one it did not, and a line it did not report at all + _line(2, 1, [0, 1, 1], [3, 5, 9]), + _line(5, 1), + _line(7, 1), + ], + TEST_LANG_PYTHON, + ) + ] + ) + ) + + file = coverage.file_find("test/lib/common/log.py") + + # The line the second run added is in place, in line order + assert_equal([(line.no, line.hit) for line in file.line_list], [(2, 2), (5, 1), (7, 1)]) + + # The arc neither run took is still missing and the one only the second run knew about is there + assert_equal(file.line_list[0].target_list, [3, 5, 9]) + assert_equal(file.line_list[0].branch_list, [1, 1, 1]) + + +#################################################################################################################################### +def test_coverage_merge_python_branch(): + """Branches arrive whichever way round the runs come back.""" + + # A run with no branches on a line takes the branches the other run reported + coverage = _coverage([_file("test/lib/common/log.py", [_line(2, 1)], TEST_LANG_PYTHON)]) + coverage.merge(_coverage([_file("test/lib/common/log.py", [_line(2, 1, [1, 0], [3, 5])], TEST_LANG_PYTHON)])) + + file = coverage.file_find("test/lib/common/log.py") + + assert_equal(file.line_list[0].branch_list, [1, 0]) + assert_equal(file.line_list[0].target_list, [3, 5]) + + # A run with branches is not given up when the other run reported none + coverage = _coverage([_file("test/lib/common/log.py", [_line(2, 1, [1, 0], [3, 5])], TEST_LANG_PYTHON)]) + coverage.merge(_coverage([_file("test/lib/common/log.py", [_line(2, 1)], TEST_LANG_PYTHON)])) + + file = coverage.file_find("test/lib/common/log.py") + + assert_equal(file.line_list[0].branch_list, [1, 0]) + assert_equal(file.line_list[0].target_list, [3, 5]) + + +#################################################################################################################################### +def test_coverage_module_file(): + """A code module is mapped to the file coverage reports it under, which depends on the language of the test.""" + + # Project source + assert_equal(coverage_module_file("common/error"), "src/common/error.c") + + # The harness and the documentation tool live beside their own source + assert_equal(coverage_module_file("test/common/harnessLog"), "test/src/common/harnessLog.c") + assert_equal(coverage_module_file("doc/command/build/build"), "doc/src/command/build/build.c") + + # A module that is included rather than compiled + assert_equal(coverage_module_file("command/backup/process.inc"), "src/command/backup/process.c.inc") + assert_equal(coverage_module_file("common/regExp.vendor"), "src/common/regExp.vendor.c.inc") + + # The python the harness is written in lives in the library rather than beside the tests + assert_equal(coverage_module_file("test/common/string_id", TEST_LANG_PYTHON), "test/lib/common/string_id.py") + + with assert_raises(TestError) as error: + coverage_module_file("common/string_id", TEST_LANG_PYTHON) + + assert_equal(str(error.exception), "python module 'common/string_id' must be under test/") + + +#################################################################################################################################### +def _cmd_coverage(path, module_list, raw_map, vm="none", coverage_summary=False, define=DEFINE): + """Run the coverage command over a repository built from the raw coverage given. + + The command reads the raw coverage from the repository and the source from the copy the tests ran against, so both are written + here the way a run leaves them.""" + + path_repo = os.path.join(path, "repo") + path_raw = os.path.join(path_repo, "test/result/coverage/raw") + + os.makedirs(path_raw, exist_ok=True) + os.makedirs(os.path.join(path_repo, "src/common"), exist_ok=True) + os.makedirs(os.path.join(path_repo, "test/lib/common"), exist_ok=True) + + for name, content in ( + ("test/define.yaml", define), + ("src/common/error.c", SOURCE_C), + ("src/common/stackTrace.c", SOURCE_C), + ("test/lib/common/log.py", SOURCE_PY), + ): + with open(os.path.join(path_repo, name), "w") as file: + file.write(content) + + for name, raw in raw_map.items(): + with open(os.path.join(path_raw, name), "w") as file: + file.write(json.dumps(raw)) + + output = io.StringIO() + + log_init(WARN, False) + + try: + with redirect_stdout(output): + result = cmd_coverage(Config(path_repo, path, module_list, vm, coverage_summary)) + finally: + log_init(INFO, True) + + return result, output.getvalue(), path_repo + + +#################################################################################################################################### +def _raw_c(name, line_list): + """Build gcov json, which reports files as an array.""" + + return {"files": [{"file": "/repo/" + name, "functions": [], "lines": line_list}]} + + +#################################################################################################################################### +def _raw_py(name, executed, missing, executed_branch=(), missing_branch=()): + """Build coverage.py json, which reports files as an object keyed by path.""" + + return { + "files": { + "/repo/" + + name: { + "executed_lines": list(executed), + "missing_lines": list(missing), + "executed_branches": [list(arc) for arc in executed_branch], + "missing_branches": [list(arc) for arc in missing_branch], + } + } + } + + +#################################################################################################################################### +def test_coverage_command(): + """Both languages are read into one report, which is written where the documentation build looks for it.""" + + with tempfile.TemporaryDirectory() as path: + result, output, path_repo = _cmd_coverage( + path, + ["common/error", "test/common/log"], + { + "test-common-error.json": _raw_c( + "src/common/error.c", [{"line_number": 5, "count": 1}, {"line_number": 6, "count": 1}] + ), + "test-test-common-log.json": _raw_py("test/lib/common/log.py", [1, 2, 3, 5], [], [[2, 3], [2, 5]]), + }, + ) + + assert_equal(result, 0) + assert_equal(output, "") + + # The report is written for every module, whether or not it had anything missing + report = open(os.path.join(path_repo, "test/result/coverage/coverage.html")).read() + + assert_in("src/common/error.c", report) + assert_in("test/lib/common/log.py", report) + + +#################################################################################################################################### +def test_coverage_command_missing(): + """A module that is not fully covered, or that produced nothing at all, is reported and fails the run.""" + + with tempfile.TemporaryDirectory() as path: + result, output, path_repo = _cmd_coverage( + path, + ["common/error", "test/common/log"], + { + "test-common-error.json": _raw_c( + "src/common/error.c", [{"line_number": 5, "count": 0}, {"line_number": 6, "count": 1}] + ), + }, + ) + + # A module with a line that never ran + assert_equal(result, 1) + assert_in("module 'src/common/error.c' is not fully covered (1/2 lines, 0/0 branches)", output) + + # A module that produced no coverage at all, which would otherwise pass silently since there is nothing to report + assert_in("module 'test/lib/common/log.py' has no coverage data", output) + + # What is missing is in the report, and the line that ran is context rather than a row of its own + report = open(os.path.join(path_repo, "test/result/coverage/coverage.html")).read() + + assert_in("report-table-row-code-uncovered", report) + + +#################################################################################################################################### +def test_coverage_command_partial(): + """A module covered by a test that was not run cannot be reported on, since it would look uncovered.""" + + with tempfile.TemporaryDirectory() as path: + result, output, path_repo = _cmd_coverage( + path, + ["common/error"], + { + "test-common-error.json": _raw_c("src/common/error.c", [{"line_number": 5, "count": 1}]), + }, + define=DEFINE.replace(" - common/stackTrace", " - common/error"), + ) + + assert_equal(result, 0) + assert_in("module 'common/error' did not have all tests run required for coverage", output) + + +#################################################################################################################################### +def test_coverage_command_summary(): + """The summary the documentation builds from is written only when it is asked for, and only from a real run.""" + + with tempfile.TemporaryDirectory() as path: + result, output, path_repo = _cmd_coverage( + path, + ["common/error"], + {"test-common-error.json": _raw_c("src/common/error.c", [{"line_number": 5, "count": 1}])}, + vm="u22", + coverage_summary=True, + ) + + assert_equal(result, 0) + + summary = open(os.path.join(path_repo, "doc/xml/auto/metric-coverage-report.auto.xml")).read() + + assert_in("common", summary) + assert_in("TOTAL", summary) + + # A summary from a run outside a vm would report code as uncovered that a vm covers + with assert_raises(TestError) as error: + _cmd_coverage( + path, + ["common/error"], + {"test-common-error.json": _raw_c("src/common/error.c", [{"line_number": 5, "count": 1}])}, + coverage_summary=True, + ) + + assert_equal(str(error.exception), "coverage summary must be run in vm") diff --git a/test/src/module/test/command/coverage/filter_c_test.py b/test/src/module/test/command/coverage/filter_c_test.py new file mode 100644 index 000000000..af9ce34e2 --- /dev/null +++ b/test/src/module/test/command/coverage/filter_c_test.py @@ -0,0 +1,176 @@ +"""Test C Coverage Filter.""" + +#################################################################################################################################### +from harness.test import * + +from command.coverage.coverage import Coverage, CoverageFile, CoverageLine +from command.coverage.filter_c import * +from command.test.define import TEST_LANG_PYTHON + +# Source the annotations are read from. The debug logging and the assertion are what the filter drops so the report is about code +# that matters rather than about macros that can never go both ways. +SOURCE = ( + "/***/\n" # 1 + "void\n" # 2 + "funcOne(bool test)\n" # 3 + "{\n" # 4 + " FUNCTION_LOG_BEGIN(logLevelDebug);\n" # 5 + " FUNCTION_LOG_PARAM(BOOL, test);\n" # 6 + " FUNCTION_LOG_END();\n" # 7 + "\n" # 8 + " ASSERT(test != NULL);\n" # 9 + "\n" # 10 + " if (test) // {uncovered_branch}\n" # 11 + " doSomething(); // {uncovered}\n" # 12 + "\n" # 13 + " doOther(); // {vm_covered}\n" # 14 + "\n" # 15 + " FUNCTION_LOG_RETURN_VOID();\n" # 16 + "}\n" # 17 + "\n" # 18 +) + + +#################################################################################################################################### +def _coverage_build(): + """Build the coverage a run of the source above would produce.""" + + result = Coverage() + + file = CoverageFile("src/common/error.c") + result.file_list.append(file) + + for no, hit, branch_list in ( + (5, 1, None), + (6, 1, None), + (7, 1, None), + (9, 1, [1, 0]), + (11, 1, [1, 0]), + (12, 0, None), + (14, 0, None), + (16, 1, None), + ): + file.line_list.append(CoverageLine(no, hit, branch_list)) + + return result, file + + +#################################################################################################################################### +def test_filter_c_read(): + """Only the files being reported on are read, with the functions and the branches on each line.""" + + coverage = coverage_read( + { + "files": [ + { + "file": "/test/repo/src/common/error.c", + "functions": [{"name": "errorNew", "start_line": 10, "end_line": 20, "execution_count": 3}], + "lines": [ + {"line_number": 10, "count": 3, "branches": []}, + {"line_number": 12, "count": 3, "branches": [{"count": 3}, {"count": 0}]}, + {"line_number": 14, "count": 0}, + ], + }, + # A shimmed .c.inc is compiled from the copy in the unit path, which is not under src + { + "file": "/test/unit-0/none/command/backup/process.c.inc", + "functions": [], + "lines": [{"line_number": 5, "count": 1, "branches": []}], + }, + # A file that is compiled but is not one of the modules being reported on + {"file": "/test/repo/src/common/other.c", "functions": [], "lines": [{"line_number": 1, "count": 1}]}, + ] + }, + ["src/common/error.c", "src/command/backup/process.c.inc"], + ) + + assert_equal([file.name for file in coverage.file_list], ["src/command/backup/process.c.inc", "src/common/error.c"]) + + file = coverage.file_list[1] + + # A function is reported with the lines it spans and how many times it was called + assert_equal([(f.name, f.line_begin, f.line_end, f.hit) for f in file.function_list], [("errorNew", 10, 20, 3)]) + + # C records how many times a line ran, and a line with no branches has none + assert_equal([(line.no, line.hit) for line in file.line_list], [(10, 3), (12, 3), (14, 0)]) + assert_is_none(file.line_list[0].branch_list) + assert_equal(file.line_list[1].branch_list, [3, 0]) + assert_is_none(file.line_list[2].branch_list) + + # A C branch has no target, which is what tells the report to mark each outcome rather than name a line + assert_is_none(file.line_list[1].target_list) + + +#################################################################################################################################### +def test_filter_c_log(): + """Debug logging that ran is dropped, since it would otherwise be most of the report.""" + + coverage, file = _coverage_build() + + coverage_filter(coverage, lambda name: SOURCE, False, "none") + + # The log begin, param, end, and the return that is the last thing in the function are all gone + assert_equal([line.no for line in file.line_list], [9, 11, 12, 14]) + + +#################################################################################################################################### +def test_filter_c_exception(): + """A branch that can never go both ways is dropped and an annotated line is treated as covered.""" + + coverage, file = _coverage_build() + + coverage_filter(coverage, lambda name: SOURCE, False, "none") + + # An assertion only fails when the code is wrong, so its branch is not something a test can cover + assert_equal(file.line_list[0].no, 9) + assert_is_none(file.line_list[0].branch_list) + + # A branch annotated in the source is dropped the same way + assert_equal(file.line_list[1].no, 11) + assert_is_none(file.line_list[1].branch_list) + + # An annotated line is treated as having run + assert_equal(file.line_list[2].no, 12) + assert_equal(file.line_list[2].hit, 1) + + # Code that only a container can reach is covered when the tests are not running in one + assert_equal(file.line_list[3].no, 14) + assert_equal(file.line_list[3].hit, 1) + + +#################################################################################################################################### +def test_filter_c_exception_vm(): + """In a vm the code a container reaches really is covered, so the annotation for it does not apply.""" + + coverage, file = _coverage_build() + + coverage_filter(coverage, lambda name: SOURCE, False, "u22") + + assert_equal(file.line_list[3].no, 14) + assert_equal(file.line_list[3].hit, 0) + + +#################################################################################################################################### +def test_filter_c_exception_summary(): + """Only uncoverable counts for the documentation summary, so the summary reports what genuinely cannot be covered.""" + + coverage, file = _coverage_build() + + # A python file in the same report is left to the python filter + file_py = CoverageFile("test/lib/common/log.py", TEST_LANG_PYTHON) + file_py.line_list.append(CoverageLine(5, 0, [0], [7])) + coverage.file_list.append(file_py) + + coverage_filter(coverage, lambda name: SOURCE, True, "u22") + + # Uncovered says the code is not covered yet rather than that it cannot be, so the summary still counts it + assert_equal(file.line_list[1].no, 11) + assert_equal(file.line_list[1].branch_list, [1, 0]) + assert_equal(file.line_list[2].hit, 0) + + # An assertion is still dropped, since that is about what a branch can do rather than about what is covered + assert_is_none(file.line_list[0].branch_list) + + # The python file is untouched + assert_equal(file_py.line_list[0].hit, 0) + assert_equal(file_py.line_list[0].branch_list, [0]) diff --git a/test/src/module/test/command/coverage/filter_py_test.py b/test/src/module/test/command/coverage/filter_py_test.py new file mode 100644 index 000000000..5104f197b --- /dev/null +++ b/test/src/module/test/command/coverage/filter_py_test.py @@ -0,0 +1,154 @@ +"""Test Python Coverage Filter.""" + +#################################################################################################################################### +from harness.test import * + +from command.coverage.coverage import Coverage, CoverageFile, CoverageLine +from command.coverage.filter_py import * +from command.test.define import TEST_LANG_C, TEST_LANG_PYTHON + +# Coverage as coverage.py reports it, i.e. the lines that ran, the lines that did not, and the arcs between them +RAW = { + "files": { + "/test/repo/test/lib/common/covered.py": { + "executed_lines": [1, 2, 3, 5], + "missing_lines": [], + "executed_branches": [[2, 3], [2, 5]], + "missing_branches": [], + }, + # A file that is measured but is not one of the modules being reported on, e.g. a module a test only depends on + "/test/repo/test/lib/common/other.py": { + "executed_lines": [1], + "missing_lines": [2], + "executed_branches": [], + "missing_branches": [], + }, + } +} + +# Source the annotations are read from +SOURCE = ( + "def func(a):\n" # 1 + " if a: # {uncovered_branch}\n" # 2 + " return 1\n" # 3 + "\n" # 4 + " return 0 # {uncovered}\n" # 5 +) + +# The same source annotated as unable to be covered rather than as not covered yet +SOURCE_UNCOVERABLE = ( + "def func(a):\n" # 1 + " if a: # {uncoverable_branch}\n" # 2 + " return 1\n" # 3 + "\n" # 4 + " return 0 # {uncoverable}\n" # 5 +) + + +#################################################################################################################################### +def test_filter_py_read(): + """Only the files being reported on are read, with the arcs out of a line collected onto it.""" + + coverage = coverage_read(RAW, ["test/lib/common/covered.py"]) + + assert_equal([file.name for file in coverage.file_list], ["test/lib/common/covered.py"]) + + file = coverage.file_list[0] + + assert_equal(file.lang, TEST_LANG_PYTHON) + + # Python records whether a line ran rather than how many times, so a hit is only ever zero or one + assert_equal([(line.no, line.hit) for line in file.line_list], [(1, 1), (2, 1), (3, 1), (5, 1)]) + + # Arcs are collected onto the line they leave, in order of where they go, and a line with no arcs has none + assert_equal(file.line_list[1].branch_list, [1, 1]) + assert_equal(file.line_list[1].target_list, [3, 5]) + assert_is_none(file.line_list[0].branch_list) + assert_is_none(file.line_list[0].target_list) + + # Function coverage is not collected since the summary it feeds covers the project source rather than the harness + assert_equal(file.function_list, []) + + +#################################################################################################################################### +def test_filter_py_read_missing(): + """A line or arc that was never reached is what the report is for.""" + + coverage = coverage_read( + { + "files": { + "/test/repo/test/lib/common/covered.py": { + "executed_lines": [1, 2, 3], + "missing_lines": [5], + "executed_branches": [[2, 3]], + "missing_branches": [[2, 5]], + } + } + }, + ["test/lib/common/covered.py"], + ) + + file = coverage.file_list[0] + + assert_equal([(line.no, line.hit) for line in file.line_list], [(1, 1), (2, 1), (3, 1), (5, 0)]) + + # The arc that was taken and the one that was not are both on the line they leave + assert_equal(file.line_list[1].branch_list, [1, 0]) + assert_equal(file.line_list[1].target_list, [3, 5]) + + +#################################################################################################################################### +def test_filter_py_exception(): + """A line or branch annotated in the source is treated as covered.""" + + coverage = Coverage() + + file = CoverageFile("test/lib/common/covered.py", TEST_LANG_PYTHON) + coverage.file_list.append(file) + + file.line_list.append(CoverageLine(2, 1, [1, 0], [3, 5])) + file.line_list.append(CoverageLine(5, 0, None)) + + # A C file in the same report is left to the C filter + file_c = CoverageFile("src/common/error.c") + file_c.line_list.append(CoverageLine(5, 0, [0, 0])) + coverage.file_list.append(file_c) + + coverage_filter(coverage, lambda name: SOURCE, False, "none") + + # The branch is removed rather than marked, so the report has nothing to show for it + assert_is_none(file.line_list[0].branch_list) + assert_is_none(file.line_list[0].target_list) + + # The line is marked as having run + assert_equal(file.line_list[1].hit, 1) + + # The C file is untouched + assert_equal(file_c.line_list[0].hit, 0) + assert_equal(file_c.line_list[0].branch_list, [0, 0]) + + +#################################################################################################################################### +def test_filter_py_exception_summary(): + """Only uncoverable counts for the documentation summary, so the summary reports what genuinely cannot be covered.""" + + coverage = Coverage() + + file = CoverageFile("test/lib/common/covered.py", TEST_LANG_PYTHON) + coverage.file_list.append(file) + + file.line_list.append(CoverageLine(2, 1, [1, 0], [3, 5])) + file.line_list.append(CoverageLine(5, 0, None)) + + coverage_filter(coverage, lambda name: SOURCE, True, "u22") + + # Uncovered says the code is not covered yet rather than that it cannot be, so the summary still counts it + assert_equal(file.line_list[0].branch_list, [1, 0]) + assert_equal(file.line_list[0].target_list, [3, 5]) + assert_equal(file.line_list[1].hit, 0) + + # Uncoverable is what the summary honors + coverage_filter(coverage, lambda name: SOURCE_UNCOVERABLE, True, "u22") + + assert_is_none(file.line_list[0].branch_list) + assert_equal(file.line_list[1].hit, 1) diff --git a/test/src/module/test/command/coverage/report_test.py b/test/src/module/test/command/coverage/report_test.py new file mode 100644 index 000000000..a13da6f36 --- /dev/null +++ b/test/src/module/test/command/coverage/report_test.py @@ -0,0 +1,246 @@ +"""Test Coverage Report. + +The whole report is asserted rather than parts of it, since the report is the output of the coverage command and a change anywhere +in it should be deliberate. The constants the renderer builds from are used to write the expected report, which keeps the assertion +about the structure rather than about the styling.""" + +#################################################################################################################################### +from harness.test import * + +from command.coverage.coverage import Coverage, CoverageFile, CoverageLine +from command.coverage.report import * +from command.test.define import TEST_LANG_PYTHON + +# C source to report against. The blank lines and braces are what the context trimming removes, and the gap between the functions is +# what puts a skip in the report. +SOURCE_C = ( + "/***/\n" # 1 + "void\n" # 2 + "funcOne(bool test)\n" # 3 + "{\n" # 4 + " if (test)\n" # 5 + " {\n" # 6 + " doSomething();\n" # 7 + " }\n" # 8 + "\n" # 9 + " return;\n" # 10 + "}\n" # 11 + "\n" # 12 + "\n" # 13 + "void\n" # 14 + "funcTwo(void)\n" # 15 + "{\n" # 16 + "\n" # 17 + " doOther();\n" # 18 + "}\n" # 19 + "\n" # 20 + "\n" # 21 + "\n" # 22 + "\n" # 23 + "void\n" # 24 + "funcThree(void)\n" # 25 + "{\n" # 26 + "\n" # 27 + "\n" # 28 + "\n" # 29 + "\n" # 30 + " doThird();\n" # 31 + "}\n" # 32 + "\n" # 33 +) + +# Python source to report against +SOURCE_PY = ( + "def func(a, b):\n" # 1 + " if a:\n" # 2 + " return 1\n" # 3 + "\n" # 4 + " for item in b:\n" # 5 + " print(item)\n" # 6 + "\n" # 7 + " return 0\n" # 8 + "\n" # 9 + "\n" # 10 + "def other(a):\n" # 11 + " if a:\n" # 12 + " return 1\n" # 13 +) + + +#################################################################################################################################### +def _source_read(name): + """Read the source a file is reported against, which the report command reads from the repository copy.""" + + return SOURCE_PY if name.endswith(".py") else SOURCE_C + + +#################################################################################################################################### +def test_report_render(): + """A file with no lines left is listed but has no report, and one that is missing coverage is reported with its context.""" + + coverage = Coverage() + + # A file that is fully covered has had every line removed by the time the report is rendered + coverage.file_list.append(CoverageFile("src/common/covered.c")) + + file = CoverageFile("src/common/missing.c") + coverage.file_list.append(file) + + # A line where one of the two branches was never taken, a line that never ran, and a line in the function below + file.line_list.append(CoverageLine(5, 1, [1, 0])) + file.line_list.append(CoverageLine(7, 0, None)) + file.line_list.append(CoverageLine(18, 0, None)) + file.line_list.append(CoverageLine(31, 0, None)) + + expect = HTML_PRE + TOC_PRE + + # A covered file is named in the contents but is not a link, since there is nothing to link to + expect += TOC_COVERED_PRE + "src/common/covered.c" + TOC_COVERED_POST + expect += TOC_UNCOVERED_PRE + "src/common/missing.c" + TOC_UNCOVERED_MID + "src/common/missing.c" + TOC_UNCOVERED_POST + expect += TOC_POST + + expect += RPT_PRE + "src/common/missing.c" + RPT_MID1 + "src/common/missing.c" + RPT_MID2 + + # Context above the first line reported, which starts at the top of the file since the line is within five of it + expect += RPT_LINE_PRE + "1" + RPT_BRANCH_COVERED + RPT_CODE + "/***/" + RPT_LINE_POST + expect += RPT_LINE_PRE + "2" + RPT_BRANCH_COVERED + RPT_CODE + "void" + RPT_LINE_POST + expect += RPT_LINE_PRE + "3" + RPT_BRANCH_COVERED + RPT_CODE + "funcOne(bool test)" + RPT_LINE_POST + expect += RPT_LINE_PRE + "4" + RPT_BRANCH_COVERED + RPT_CODE + "{" + RPT_LINE_POST + + # The line ran so only the branch is marked, with one mark per outcome + expect += RPT_LINE_PRE + "5" + RPT_BRANCH_UNCOVERED_PRE + "[+ -]" + RPT_BRANCH_UNCOVERED_POST + expect += RPT_CODE + " if (test)" + RPT_LINE_POST + + # The context between the two lines is the brace, which is not trimmed here since it is above a line being reported + expect += RPT_LINE_PRE + "6" + RPT_BRANCH_COVERED + RPT_CODE + " {" + RPT_LINE_POST + + # The line never ran so the code is marked rather than the branch + expect += RPT_LINE_PRE + "7" + RPT_BRANCH_COVERED + RPT_CODE_UNCOVERED + " doSomething();" + RPT_LINE_POST + + # Context below, with the closing brace and the blank line at the end of the function trimmed off + expect += RPT_LINE_PRE + "8" + RPT_BRANCH_COVERED + RPT_CODE + " }" + RPT_LINE_POST + expect += RPT_LINE_PRE + "9" + RPT_BRANCH_COVERED + RPT_CODE + "" + RPT_LINE_POST + expect += RPT_LINE_PRE + "10" + RPT_BRANCH_COVERED + RPT_CODE + " return;" + RPT_LINE_POST + + # The next line reported is far enough below that the report skips rather than running the context together, and the blank line + # at the top of that context is trimmed + expect += RPT_SKIP + expect += RPT_LINE_PRE + "14" + RPT_BRANCH_COVERED + RPT_CODE + "void" + RPT_LINE_POST + expect += RPT_LINE_PRE + "15" + RPT_BRANCH_COVERED + RPT_CODE + "funcTwo(void)" + RPT_LINE_POST + expect += RPT_LINE_PRE + "16" + RPT_BRANCH_COVERED + RPT_CODE + "{" + RPT_LINE_POST + expect += RPT_LINE_PRE + "17" + RPT_BRANCH_COVERED + RPT_CODE + "" + RPT_LINE_POST + expect += RPT_LINE_PRE + "18" + RPT_BRANCH_COVERED + RPT_CODE_UNCOVERED + " doOther();" + RPT_LINE_POST + + # Nothing but a brace and blank lines above the next line reported, so the whole context is trimmed and the skip stands alone + expect += RPT_SKIP + expect += RPT_LINE_PRE + "31" + RPT_BRANCH_COVERED + RPT_CODE_UNCOVERED + " doThird();" + RPT_LINE_POST + + expect += RPT_POST + HTML_POST + + assert_equal(report_render(coverage, _source_read), expect) + + +#################################################################################################################################### +def test_report_render_python(): + """A python branch is an arc to another line, so the report names the line that was never jumped to.""" + + coverage = Coverage() + + file = CoverageFile("test/lib/common/py.py", TEST_LANG_PYTHON) + coverage.file_list.append(file) + + # One of the two arcs was taken, then neither of them + file.line_list.append(CoverageLine(2, 1, [1, 0], [3, 5])) + file.line_list.append(CoverageLine(5, 1, [0, 0], [6, 8])) + + # An arc out of the function it is in, which python reports as the line the function is defined on, negated + file.line_list.append(CoverageLine(12, 1, [0, 1], [-11, 13])) + + expect = HTML_PRE + TOC_PRE + expect += TOC_UNCOVERED_PRE + "test/lib/common/py.py" + TOC_UNCOVERED_MID + "test/lib/common/py.py" + TOC_UNCOVERED_POST + expect += TOC_POST + + expect += RPT_PRE + "test/lib/common/py.py" + RPT_MID1 + "test/lib/common/py.py" + RPT_MID2 + + expect += RPT_LINE_PRE + "1" + RPT_BRANCH_COVERED + RPT_CODE + "def func(a, b):" + RPT_LINE_POST + + # Only the arc that was never taken is named, i.e. the branch that returns rather than the one that falls through + expect += RPT_LINE_PRE + "2" + RPT_BRANCH_UNCOVERED_PRE + "[-> 5]" + RPT_BRANCH_UNCOVERED_POST + expect += RPT_CODE + " if a:" + RPT_LINE_POST + + expect += RPT_LINE_PRE + "3" + RPT_BRANCH_COVERED + RPT_CODE + " return 1" + RPT_LINE_POST + expect += RPT_LINE_PRE + "4" + RPT_BRANCH_COVERED + RPT_CODE + "" + RPT_LINE_POST + + # Both arcs are named when neither was taken, i.e. a loop that never ran and never finished + expect += RPT_LINE_PRE + "5" + RPT_BRANCH_UNCOVERED_PRE + "[-> 6 -> 8]" + RPT_BRANCH_UNCOVERED_POST + expect += RPT_CODE + " for item in b:" + RPT_LINE_POST + + expect += RPT_LINE_PRE + "6" + RPT_BRANCH_COVERED + RPT_CODE + " print(item)" + RPT_LINE_POST + expect += RPT_LINE_PRE + "7" + RPT_BRANCH_COVERED + RPT_CODE + "" + RPT_LINE_POST + expect += RPT_LINE_PRE + "8" + RPT_BRANCH_COVERED + RPT_CODE + " return 0" + RPT_LINE_POST + + expect += RPT_LINE_PRE + "9" + RPT_BRANCH_COVERED + RPT_CODE + "" + RPT_LINE_POST + expect += RPT_LINE_PRE + "10" + RPT_BRANCH_COVERED + RPT_CODE + "" + RPT_LINE_POST + expect += RPT_LINE_PRE + "11" + RPT_BRANCH_COVERED + RPT_CODE + "def other(a):" + RPT_LINE_POST + + # The arc out of the function is named as a return rather than by the line it goes to, which means nothing to a reader + expect += RPT_LINE_PRE + "12" + RPT_BRANCH_UNCOVERED_PRE + "[-> return]" + RPT_BRANCH_UNCOVERED_POST + expect += RPT_CODE + " if a:" + RPT_LINE_POST + + expect += RPT_LINE_PRE + "13" + RPT_BRANCH_COVERED + RPT_CODE + " return 1" + RPT_LINE_POST + + expect += RPT_POST + HTML_POST + + assert_equal(report_render(coverage, _source_read), expect) + + +#################################################################################################################################### +def test_report_summary_value(): + """A value is rendered as hit over total with the percentage, the same way the C code renders it.""" + + # Nothing to measure is not zero percent + assert_equal(summary_value(0, 0), "---") + + # A ratio of one is fixed at 100% so rounding cannot make it anything else + assert_equal(summary_value(5, 5), "5/5 (100.00%)") + + # Nothing hit is zero, padded out to the same shape + assert_equal(summary_value(0, 5), "0/5 (0.00%)") + + # Rounding down and up + assert_equal(summary_value(1, 3), "1/3 (33.33%)") + assert_equal(summary_value(2, 3), "2/3 (66.67%)") + assert_equal(summary_value(3, 4), "3/4 (75.00%)") + + +#################################################################################################################################### +def test_report_summary_render(): + """The summary aggregates the files of a code module into one row and covers only project source.""" + + coverage = Coverage() + + for name, value in ( + ("src/common/error.c", (2, 2, 3, 4, 10, 10)), + ("src/common/log.c", (1, 2, 0, 0, 5, 6)), + ("src/config/parse.c", (4, 4, 2, 2, 20, 20)), + # The harness and the documentation tool are covered but are not what the summary reports on + ("test/src/common/harness.c", (9, 9, 9, 9, 9, 9)), + ("doc/src/build.c", (9, 9, 9, 9, 9, 9)), + ): + file = CoverageFile(name) + ( + file.function_hit, + file.function_total, + file.branch_hit, + file.branch_total, + file.line_hit, + file.line_total, + ) = value + + coverage.file_list.append(file) + + expect = SUMMARY_ROW % ("common", "3/4 (75.00%)", "3/4 (75.00%)", "15/16 (93.75%)") + "\n" + expect += SUMMARY_ROW % ("config", "4/4 (100.00%)", "2/2 (100.00%)", "20/20 (100.00%)") + "\n" + expect += SUMMARY_ROW % ("TOTAL", "7/8 (87.50%)", "5/6 (83.33%)", "35/36 (97.22%)") + + assert_equal(summary_render(coverage), expect) diff --git a/test/src/module/test/command/lint/lint_test.py b/test/src/module/test/command/lint/lint_test.py new file mode 100644 index 000000000..9c9c51fcf --- /dev/null +++ b/test/src/module/test/command/lint/lint_test.py @@ -0,0 +1,229 @@ +"""Test Code Linter. + +The linter is made of the scan for content that could hide code and the check that StringId macros encode what they claim to, so +both are tested here along with the command that walks the repository and applies them.""" + +#################################################################################################################################### +import io +import os +import tempfile +from contextlib import redirect_stdout + +from harness.test import * + +from command.lint.ascii import * +from command.lint.lint import * +from command.lint.string_id import * +from common.error import * +from common.log import * + + +#################################################################################################################################### +def _capture(function): + """Run a function and return what it returned along with what it reported. + + Timestamps are suppressed so the report is exact and the log settings are put back afterwards.""" + + output = io.StringIO() + + log_init(WARN, False) + + try: + with redirect_stdout(output): + result = function() + finally: + log_init(INFO, True) + + return result, output.getvalue() + + +#################################################################################################################################### +def _lint(file_map, symlink=False): + """Lint a repository built from the files given, returning what was reported and the error raised, if any.""" + + with tempfile.TemporaryDirectory() as path: + for name, content in file_map.items(): + os.makedirs(os.path.dirname(os.path.join(path, name)), exist_ok=True) + + with open(os.path.join(path, name), "wb") as file: + file.write(content) + + # A link to a file that is not there, which is the readable case that can be provoked without changing permissions + if symlink: + os.symlink("missing.txt", os.path.join(path, "dangling.txt")) + + def run(): + try: + cmd_lint(path) + except TestError as error: + return str(error) + + return None + + return _capture(run) + + +#################################################################################################################################### +def test_lint_ascii(): + """A character outside 7-bit ASCII is reported with the line it is on.""" + + # Nothing to report for source that is already 7-bit ASCII + assert_equal(_capture(lambda: lint_ascii(b"clean text\twith a tab\n")), (0, "")) + + # A right single quote, which is what a word processor turns an apostrophe into. It is written as bytes since the source this + # test lives in must itself be 7-bit ASCII. + result, output = _capture(lambda: lint_ascii(b"line one\nit\xe2\x80\x99s\n")) + + assert_equal(result, 1) + assert_in("line 2 contains disallowed character U+2019", output) + + # Every shape the decoder reports on, i.e. four byte, two byte, a byte that starts no sequence at all, and a sequence cut short + # by the end of the file + result, output = _capture(lambda: lint_ascii(b"\xf0\x9f\x98\x80\n\xc3\xa9\n\x80\n\xe2\x80")) + + assert_equal(result, 4) + assert_in("line 1 contains disallowed character U+1F600", output) + assert_in("line 2 contains disallowed character U+00E9", output) + assert_in("line 3 contains disallowed character U+0080", output) + assert_in("line 4 contains disallowed character U+0080", output) + + +#################################################################################################################################### +def test_lint_str_id_valid(): + """A macro that encodes what it claims to is not reported.""" + + result, output = _capture( + lambda: lint_str_id( + '#define ANY STRID5("any", 0x65c10)\n' + '#define LZ4 STRID6("lz4", 0x2068c1)\n' + '#define ASC STRID5S("asc", 1, 0xe614)\n' + '#define TIME STRID5S("time", 6, 0x56a680e)\n' + ) + ) + + assert_equal(result, 0) + assert_equal(output, "") + + # Source with no macros at all has nothing to check + assert_equal(_capture(lambda: lint_str_id("int main(void) { return 0; }\n")), (0, "")) + + +#################################################################################################################################### +def test_lint_str_id_skip(): + """What is not an encoded string is skipped.""" + + # The macro definitions themselves, which name their parameters + result, output = _capture( + lambda: lint_str_id( + "#define STRID5(str, strId) strId\n" + "#define STRID5S(str, seq, strId) strId\n" + "#define STRID6(str, strId) strId\n" + "#define STRID6S(str, seq, strId) strId\n" + ) + ) + + assert_equal(result, 0) + assert_equal(output, "") + + # A quoted string inside a string, i.e. a macro in a test that is checking the macro + assert_equal(_capture(lambda: lint_str_id('TEST_RESULT_Z(text, "STRID5(\\"any\\", 0x1)", "render");\n')), (0, "")) + + # A value that is itself defined by the test + assert_equal(_capture(lambda: lint_str_id('TEST_RESULT_UINT(value, STRID5("any", TEST_VALUE), "encode");\n')), (0, "")) + + +#################################################################################################################################### +def test_lint_str_id_error(): + """A macro that does not encode what it claims to is reported, and every macro is checked rather than only the first.""" + + # The value does not match the string + result, output = _capture(lambda: lint_str_id('#define ANY STRID5("any", 0x1)\n')) + + assert_equal(result, 1) + assert_in("""'STRID5("any", 0x1)' should be 'STRID5("any", 0x65c10)'""", output) + + # The sequence is part of what is encoded, so a value from another sequence does not match + result, output = _capture(lambda: lint_str_id('#define ASC STRID5S("asc", 2, 0xe614)\n')) + + assert_equal(result, 1) + assert_in("""'STRID5S("asc", 2, 0xe614)' should be 'STRID5S("asc", 2, 0xe616)'""", output) + + # The string parameter must be quoted, which catches a value used where a string belongs + result, output = _capture(lambda: lint_str_id("#define ANY STRID5(any, 0x65c10)\n")) + + assert_equal(result, 1) + assert_in("""'STRID5(any, 0x65c10)' must have quotes around string parameter 'any'""", output) + + # A string the encoding cannot hold, reported with what was wrong with it + result, output = _capture(lambda: lint_str_id('#define BAD STRID5("under_score", 0x1)\n')) + + assert_equal(result, 1) + assert_in("""'STRID5("under_score", 0x1)' is not valid: 'under_score' contains invalid characters""", output) + + # A macro with no value at all, which is a single parameter rather than a missing one + result, output = _capture(lambda: lint_str_id('#define ANY STRID5("any")\n')) + + assert_equal(result, 1) + assert_in("""'STRID5("any")' should be 'STRID5("any", 0x65c10)'""", output) + + # Every macro is checked, so the count is what the file needs fixed rather than what stopped the scan + result, output = _capture(lambda: lint_str_id('#define ANY STRID5("any", 0x1)\n#define LZ4 STRID6("lz4", 0x2)\n')) + + assert_equal(result, 2) + assert_in("""'STRID5("any", 0x1)' should be""", output) + assert_in("""'STRID6("lz4", 0x2)' should be""", output) + + +#################################################################################################################################### +def test_lint_clean(): + """A repository with nothing to report passes silently.""" + + error, output = _lint( + { + "README.md": b"clean text\twith a tab\n", + "src/x.c": b'#define ANY STRID5("any", 0x65c10)\n', + "src/x.h": b'#define LZ4 STRID6("lz4", 0x2068c1)\n', + "src/x.c.inc": b'#define ASC STRID5S("asc", 1, 0xe614)\n', + # Generated and vendored includes are not ours to fix, so the StringId check is not applied to them + "src/x.auto.c.inc": b'#define ANY STRID5("any", 0x1)\n', + "src/x.vendor.c.inc": b'#define ANY STRID5("any", 0x1)\n', + # A binary file that is on the skip list, which is a deliberate and reviewable decision + "doc/resource/logo.png": b"\x00binary\n", + }, + symlink=True, + ) + + assert_is_none(error) + assert_equal(output, "") + + +#################################################################################################################################### +def test_lint_binary(): + """A binary file that is not on the skip list is an unscannable place to hide code.""" + + error, output = _lint({"stray.bin": b"\x00\x01\x02"}) + + assert_equal(error, "1 linter error(s) in 'stray.bin' (see warnings above)") + assert_in("unexpected binary file", output) + + +#################################################################################################################################### +def test_lint_error(): + """Both checks are applied to C source and their errors are counted together.""" + + error, output = _lint({"doc/x.md": b"line one\nit\xe2\x80\x99s\n"}) + + assert_equal(error, "1 linter error(s) in 'doc/x.md' (see warnings above)") + assert_in("line 2 contains disallowed character U+2019", output) + + error, output = _lint({"src/x.c": b'#define ANY STRID5("any", 0x1)\n'}) + + assert_equal(error, "1 linter error(s) in 'src/x.c' (see warnings above)") + assert_in("""should be 'STRID5("any", 0x65c10)'""", output) + + # A file with both kinds of error reports what it needs fixed rather than what stopped the scan + error, output = _lint({"src/x.c": b'#define ANY STRID5("any", 0x1) // it\xe2\x80\x99s wrong\n'}) + + assert_equal(error, "2 linter error(s) in 'src/x.c' (see warnings above)") + assert_in("U+2019", output) + assert_in("should be", output) diff --git a/test/src/module/test/command/test/build_test.py b/test/src/module/test/command/test/build_test.py new file mode 100644 index 000000000..89d77ab09 --- /dev/null +++ b/test/src/module/test/command/test/build_test.py @@ -0,0 +1,682 @@ +"""Test Test Build Handler. + +The build is checked by generating a few representative tests against a repository written here and comparing the whole of what +comes out, since the generated build is the thing that has to be right and any change to it should be deliberate. + +The paths in the generated files are absolute and the user and group are whoever the tests run as, so both are replaced with tokens +before comparing, the same way the C tests do it.""" + +#################################################################################################################################### +import os +import tempfile +from unittest.mock import patch + +from harness.test import * + +from command.test.build import * +from command.test.define import test_def_find, test_def_parse +from common.error import * +from common.log import * +from common.storage import path_list_recurse + +# The repository the build reads. Only the modules a harness shims are read as source, so those are the only ones written out. +DEFINE = """ +unit: + - name: common + + test: + - name: error + total: 2 + + coverage: + - common/error + + - name: exec + total: 1 + feature: exec + + harness: + - name: config + integration: false + + shim: + - common/execOther + - name: common/exec + function: + - execOne + - execTwo + - execThree: + inc: execInc + - execFour: + inc: execInc + + coverage: + - common/exec + - common/execOther + + depend: + - common/log + + - name: after + total: 1 + harness: extra + + coverage: + - common/after + +integration: + - name: real + + test: + - name: all + total: 1 + +performance: + - name: performance + + test: + - name: type + total: 1 + define: -DNDEBUG + + coverage: + - performance/type +""" + +MESON_OPTIONS = "option('fake', type: 'boolean', value: false)\n" + +MESON = ( + "project('pgbackrest', 'c')\n" + "configuration = configuration_data()\n" + " configuration.set('HAVE_LIBBACKTRACE', true)\n" + "subdir('src')\n" + "meson.add_dist_script('test/dist.sh')\n" +) + +# A module a harness shims, with an extern function, a static function, and a function defined in an included module +SOURCE_EXEC = ( + "/***/\n" + '#include "common/execInc.c.inc"\n' + "\n" + "FN_EXTERN void\n" + "execOne(const char *command)\n" + "{\n" + " doExec();\n" + "}\n" + "\n" + "static void\n" + "execTwo(void)\n" + "{\n" + " doTwo();\n" + "}\n" +) + +SOURCE_EXEC_INC = ( + "/***/\n" + "FN_EXTERN void\n" + "execThree(\n" + " const char *command,\n" + " int flag)\n" + "{\n" + " doThree();\n" + "}\n" + "\n" + "FN_EXTERN void\n" + "execFour(void)\n" + "{\n" + " doFour();\n" + "}\n" +) + +HARNESS_CONFIG = "/***/\n{[SHIM_MODULE]}\n\nvoid\nhrnConfig(void)\n{\n}\n" +HARNESS_EXTRA = "/***/\nvoid\nhrnExtra(void)\n{\n}\n" +HARNESS_HELPER = "/***/\nvoid\nhrnHelper(void)\n{\n}\n" + +# The test.c the build fills in, holding every substitution it makes +TEST_C = ( + "/***/\n" + "{[C_TEST_DEBUG_TEST_TRACE]}\n" + "{[C_INCLUDE]}\n" + '#define TEST_PATH "{[C_TEST_PATH]}"\n' + '#define HRN_PATH "{[C_HRN_PATH]}"\n' + '#define HRN_PATH_REPO "{[C_HRN_PATH_REPO]}"\n' + '#define TEST_PROJECT_EXE "{[C_TEST_PROJECT_EXE]}"\n' + '#define TEST_PGB_PATH "{[C_TEST_PGB_PATH]}"\n' + "#define TEST_LOG_EXPECT {[C_TEST_LOG_EXPECT]}\n" + "#define LOG_LEVEL_TEST {[C_LOG_LEVEL_TEST]}\n" + "#define TEST_TIMING {[C_TEST_TIMING]}\n" + '#define TEST_ARCHITECTURE "{[C_TEST_ARCHITECTURE]}"\n' + "{[C_TEST_TZ]}\n" + "#define TEST_SCALE {[C_TEST_SCALE]}\n" + "#define TEST_CONTAINER {[C_TEST_CONTAINER]}\n" + '#define TEST_GROUP "{[C_TEST_GROUP]}"\n' + "#define TEST_GROUP_ID {[C_TEST_GROUP_ID]}\n" + '#define TEST_USER "{[C_TEST_USER]}"\n' + "#define TEST_USER_LEN {[C_TEST_USER_LEN]}\n" + "#define TEST_USER_ID {[C_TEST_USER_ID]}\n" + '#define TEST_VM "{[C_TEST_VM]}"\n' + '#define TEST_PG_VERSION "{[C_TEST_PG_VERSION]}"\n' + "#define TEST_IDX {[C_TEST_IDX]}\n" + "{[C_TEST_INCLUDE]}\n" + '#define TEST_PATH_BUILD "{[C_TEST_PATH_BUILD]}"\n' + "\n" + "void\n" + "testRun(void)\n" + "{\n" + " {[C_TEST_LIST]}\n" + "}\n" +) + +# The user and group the generated test.c is built with, so it does not depend on who ran the tests +USER = "pgbackrest" +GROUP = "pgbackrest" +USER_ID = 1000 +GROUP_ID = 1001 + + +#################################################################################################################################### +class Config: + """What the build reads from the command line.""" + + def __init__(self, repo_path, test_path, **option): + self.repo_path = repo_path + self.test_path = test_path + self.vm = "none" + self.vm_id = 0 + self.pg_version = "invalid" + self.test = None + self.scale = 1 + self.tz = None + self.log_level_test = OFF + self.log_timestamp = True + self.coverage = True + self.back_trace = True + self.optimize = False + self.profile = False + + for name, value in option.items(): + setattr(self, name, value) + + +#################################################################################################################################### +def _repo_create(path): + """Write the repository the build reads.""" + + result = os.path.join(path, "repo") + + for name, content in ( + ("meson_options.txt", MESON_OPTIONS), + ("meson.build", MESON), + ("test/define.yaml", DEFINE), + ("test/src/test.c", TEST_C), + ("src/common/exec.c", SOURCE_EXEC), + ("src/common/execInc.c.inc", SOURCE_EXEC_INC), + ("test/src/harness/config.c", HARNESS_CONFIG), + ("test/src/harness/config/helper.c", HARNESS_HELPER), + ("test/src/harness/extra.c", HARNESS_EXTRA), + ): + os.makedirs(os.path.dirname(os.path.join(result, name)), exist_ok=True) + + with open(os.path.join(result, name), "w") as file: + file.write(content) + + return result + + +#################################################################################################################################### +def _build(path, module_name, architecture="x86_64", **option): + """Generate the unit build for a module, returning the unit path and the files that were generated.""" + + path_repo = os.path.join(path, "repo") + config = Config(path_repo, path, **option) + module = test_def_find(test_def_parse(path_repo), module_name) + + # The user and group are whoever the tests run as, so they are fixed here to keep the generated test.c the same everywhere + with patch("command.test.build.user_name", lambda: USER), patch("command.test.build.group_name", lambda: GROUP), patch( + "command.test.build.os.getuid", lambda: USER_ID + ), patch("command.test.build.os.getgid", lambda: GROUP_ID): + test_build = TestBuild(config, module, architecture) + test_build.build() + + return test_build.path_unit, path_list_recurse(test_build.path_unit) + + +#################################################################################################################################### +def _read(path, path_unit, name): + """Read a generated file with the paths that change from run to run replaced by tokens.""" + + with open(os.path.join(path_unit, name)) as file: + content = file.read() + + return content.replace(os.path.join(path, "repo"), "[REPO_PATH]").replace(path, "[TEST_PATH]") + + +#################################################################################################################################### +# The part of meson.build that the build appends, which is the same for every module apart from the arguments and the sources +MESON_CONFIGURE = ( + "\n" + + MESON_COMMENT_BLOCK + + "\n# Write configuration\n" + + MESON_COMMENT_BLOCK + + "\n" + + "configure_file(output: 'build.auto.h', configuration: configuration)\n" + + "\n" + + "add_global_arguments('-DFN_EXTERN=extern', language : 'c')\n" + + "add_global_arguments('-DVR_EXTERN_DECLARE=extern', language : 'c')\n" + + "add_global_arguments('-DVR_EXTERN_DEFINE=', language : 'c')\n" + + "add_global_arguments('-DERROR_MESSAGE_BUFFER_SIZE=131072', language : 'c')\n" +) + +MESON_UNIT = "\n" + MESON_COMMENT_BLOCK + "\n# Unit test\n" + MESON_COMMENT_BLOCK + "\n" + "src_unit = files(\n" + +MESON_EXECUTABLE = ( + " '../../repo/test/src/harness/test.c',\n" + " 'test.c',\n" + ")\n" + "\n" + "executable(\n" + " 'test-unit',\n" + " sources: src_unit,\n" +) + +MESON_INCLUDE = ( + " include_directories:\n" + " include_directories(\n" + " '.',\n" + " '../../repo/src',\n" + " '../../repo/doc/src',\n" + " '../../repo/test/src',\n" + " ),\n" + " dependencies: [\n" +) + +MESON_LIB = ( + " lib_bz2,\n" + " lib_openssl,\n" + " lib_lz4,\n" + " lib_pq,\n" + " lib_ssh2,\n" + " lib_xml,\n" + " lib_yaml,\n" + " lib_z,\n" + " lib_zstd,\n" + " ],\n" + ")\n" +) + +# meson.build as the build leaves the part it read from the repository +MESON_READ = ( + "project('pgbackrest', 'c')\n" + "configuration = configuration_data()\n" + " configuration.set('HAVE_LIBBACKTRACE', true)\n" + "# subdir('src')\n" + "# meson.add_dist_script('test/dist.sh')\n" +) + + +#################################################################################################################################### +def test_build_module(): + """A module with nothing to shim generates the build and the test wrapper and nothing else.""" + + with tempfile.TemporaryDirectory() as path: + _repo_create(path) + path_unit, file_list = _build(path, "common/error") + + assert_equal(file_list, ["meson.build", "meson_options.txt", "test.c"]) + + # The options are copied as they are, since the build needs them to configure + assert_equal(_read(path, path_unit, "meson_options.txt"), MESON_OPTIONS) + + # What was read from the repository, with the parts that are not built for testing commented out + assert_equal( + _read(path, path_unit, "meson.build"), + MESON_READ + + MESON_CONFIGURE + + "add_global_arguments('-DDEBUG_COVERAGE', language : 'c')\n" + + MESON_UNIT + + MESON_EXECUTABLE + + MESON_INCLUDE + + " lib_backtrace,\n" + + MESON_LIB, + ) + + # The wrapper includes the module under test and the test module, and lists the runs it may run + assert_equal( + _read(path, path_unit, "test.c"), + "/***/\n" + "#define DEBUG_TEST_TRACE\n" + '#include "../../repo/src/common/error.c"\n' + '#define TEST_PATH "[TEST_PATH]/test-0"\n' + '#define HRN_PATH "[TEST_PATH]/data-0"\n' + '#define HRN_PATH_REPO "[REPO_PATH]"\n' + '#define TEST_PROJECT_EXE "[TEST_PATH]/build/none/src/pgbackrest"\n' + '#define TEST_PGB_PATH "../../../repo"\n' + "#define TEST_LOG_EXPECT true\n" + "#define LOG_LEVEL_TEST logLevelOff\n" + "#define TEST_TIMING true\n" + '#define TEST_ARCHITECTURE "x86_64"\n' + "// No timezone specified\n" + "#define TEST_SCALE 1\n" + "#define TEST_CONTAINER false\n" + '#define TEST_GROUP "pgbackrest"\n' + "#define TEST_GROUP_ID 1001\n" + '#define TEST_USER "pgbackrest"\n' + "#define TEST_USER_LEN 10\n" + "#define TEST_USER_ID 1000\n" + '#define TEST_VM "none"\n' + '#define TEST_PG_VERSION "invalid"\n' + "#define TEST_IDX 0\n" + '#include "../../repo/test/src/module/common/errorTest.c"\n' + '#define TEST_PATH_BUILD "[TEST_PATH]/unit-0/none/build"\n' + "\n" + "void\n" + "testRun(void)\n" + "{\n" + " hrnAdd( 1, true);\n" + " hrnAdd( 2, true);\n" + "}\n", + ) + + +#################################################################################################################################### +def test_build_shim(): + """A module a harness shims is copied with the shimmed functions renamed, along with the harness that reaches into it.""" + + with tempfile.TemporaryDirectory() as path: + _repo_create(path) + path_unit, file_list = _build(path, "common/exec") + + # The shim module keeps its normal path while the included module is written where the include resolves to it + assert_equal( + file_list, + [ + "common/execInc.c.inc", + "meson.build", + "meson_options.txt", + "src/common/exec.c", + "test.c", + "test/src/harness/config.c", + ], + ) + + # An extern function is declared under the shimmed name so there is no missing prototype, a static function keeps a + # declaration under its own name so the module can still call it, and the line count is unchanged so coverage still lines up + assert_equal( + _read(path, path_unit, "src/common/exec.c"), + "/***/\n" + '#include "common/execInc.c.inc"\n' + "\n" + "FN_EXTERN void execOne_SHIMMED(const char *command); FN_EXTERN void\n" + "execOne_SHIMMED(const char *command)\n" + "{\n" + " doExec();\n" + "}\n" + "\n" + "static void execTwo(void); static void\n" + "execTwo_SHIMMED(void)\n" + "{\n" + " doTwo();\n" + "}\n", + ) + + # A signature that runs over several lines is gathered onto one for the declaration, and a module holding more than one + # shimmed function is written once with both of them renamed + assert_equal( + _read(path, path_unit, "common/execInc.c.inc"), + "/***/\n" + "FN_EXTERN void execThree_SHIMMED(const char *command, int flag); FN_EXTERN void\n" + "execThree_SHIMMED(\n" + " const char *command,\n" + " int flag)\n" + "{\n" + " doThree();\n" + "}\n" + "\n" + "FN_EXTERN void execFour_SHIMMED(void); FN_EXTERN void\n" + "execFour_SHIMMED(void)\n" + "{\n" + " doFour();\n" + "}\n", + ) + + # The harness includes the shimmed copy rather than the one in the repository, while a module it reaches into but shims no + # function in comes straight from the repository + assert_equal( + _read(path, path_unit, "test/src/harness/config.c"), + "/***/\n" + '#include "[REPO_PATH]/src/common/execOther.c"\n' + '#include "[TEST_PATH]/unit-0/none/src/common/exec.c"\n' + "\n" + "void\n" + "hrnConfig(void)\n" + "{\n" + "}\n", + ) + + # The feature the module introduces is defined, the modules it depends on are compiled in, and the harness has its own + # source compiled in. The harness itself is not, since the wrapper includes it directly. + assert_equal( + _read(path, path_unit, "meson.build"), + MESON_READ + + MESON_CONFIGURE + + "add_global_arguments('-DHRN_INTEST_EXEC', language : 'c')\n" + + "add_global_arguments('-DDEBUG_COVERAGE', language : 'c')\n" + + MESON_UNIT + + " '../../repo/src/common/error.c',\n" + + " '../../repo/src/common/log.c',\n" + + " '../../repo/test/src/harness/config/helper.c',\n" + + MESON_EXECUTABLE + + MESON_INCLUDE + + " lib_backtrace,\n" + + MESON_LIB, + ) + + # The wrapper includes the harness rather than the module, since the harness is what pulls the module in. Both covered + # modules come from the one harness so it is included once. + assert_in('#include "test/src/harness/config.c"\n', _read(path, path_unit, "test.c")) + assert_equal(_read(path, path_unit, "test.c").count("test/src/harness/config.c"), 1) + + +#################################################################################################################################### +def test_build_harness(): + """A harness a module does not reach into is compiled in, and a module a harness already includes is not compiled twice.""" + + with tempfile.TemporaryDirectory() as path: + _repo_create(path) + path_unit, file_list = _build(path, "common/after") + + meson_build = _read(path, path_unit, "meson.build") + + # The feature an earlier module introduced is available here + assert_in("add_global_arguments('-DHRN_FEATURE_EXEC', language : 'c')\n", meson_build) + + # The module this test depends on that a harness already includes is not compiled again + assert_in(" '../../repo/src/common/error.c',\n", meson_build) + assert_in(" '../../repo/src/common/log.c',\n", meson_build) + assert_not_in("src/common/exec.c',\n", meson_build) + + # A harness that reaches into a module is compiled in when this module is not the one being tested, and one that reaches + # into nothing is compiled straight from the repository + assert_in(" 'test/src/harness/config.c',\n", meson_build) + assert_in(" '../../repo/test/src/harness/extra.c',\n", meson_build) + + +#################################################################################################################################### +def test_build_integration(): + """An integration test leaves out the harnesses and shims that are only for unit tests.""" + + with tempfile.TemporaryDirectory() as path: + _repo_create(path) + path_unit, file_list = _build(path, "real/all", vm="u22") + + # Nothing was shimmed and no harness needed a copy + assert_equal(file_list, ["meson.build", "meson_options.txt", "test.c"]) + + meson_build = _read(path, path_unit, "meson.build") + + assert_not_in("harness/config", meson_build) + assert_in(" '../../repo/test/src/harness/extra.c',\n", meson_build) + + # An integration test builds outside a container even when one was named, so it is not flagged as being in one + assert_not_in("TEST_CONTAINER_REQUIRED", meson_build) + + # The expect log is only kept for unit tests, and the vm is still the one the binary was built for + test_c = _read(path, path_unit, "test.c") + + assert_in("#define TEST_LOG_EXPECT false\n", test_c) + assert_in('#define TEST_VM "u22"\n', test_c) + assert_in("#define TEST_CONTAINER false\n", test_c) + + +#################################################################################################################################### +def test_build_option(): + """The options that change how the test is compiled.""" + + with tempfile.TemporaryDirectory() as path: + _repo_create(path) + + # A performance test is timed so it is optimized, and the debug trace it would otherwise carry is left out + path_unit, file_list = _build(path, "performance/type") + meson_build = _read(path, path_unit, "meson.build") + + assert_in(" c_args: [\n '-O2',\n ],\n", meson_build) + assert_in("// Debug test trace not enabled\n", _read(path, path_unit, "test.c")) + + # A module that names a define has it set for the whole build + assert_in("add_global_arguments('-DNDEBUG', language : 'c')\n", meson_build) + + # Profiling is optimized and linked without position independence, and the debug trace is left out for the same reason + path_unit, file_list = _build(path, "common/error", profile=True, optimize=True) + meson_build = _read(path, path_unit, "meson.build") + + assert_in(" c_args: [\n '-O2',\n '-pg',\n '-no-pie',\n ],\n", meson_build) + assert_in(" link_args: [\n '-pg',\n '-no-pie',\n ],\n", meson_build) + assert_in("// Debug test trace not enabled\n", _read(path, path_unit, "test.c")) + + # Without back trace the library is not linked and the configuration that looks for it is commented out + path_unit, file_list = _build(path, "common/error", back_trace=False) + meson_build = _read(path, path_unit, "meson.build") + + assert_in("# configuration.set('HAVE_LIBBACKTRACE'", meson_build) + assert_not_in("lib_backtrace", meson_build) + + # Without coverage the define that turns it on is not set + path_unit, file_list = _build(path, "common/error", coverage=False) + + assert_not_in("DEBUG_COVERAGE", _read(path, path_unit, "meson.build")) + + # In a container the tests that need one are enabled + path_unit, file_list = _build(path, "common/error", vm="u22") + + assert_in("add_global_arguments('-DTEST_CONTAINER_REQUIRED', language : 'c')\n", _read(path, path_unit, "meson.build")) + assert_in("#define TEST_CONTAINER true\n", _read(path, path_unit, "test.c")) + + +#################################################################################################################################### +def test_build_option_test(): + """The options that change what the test runs rather than how it is compiled.""" + + with tempfile.TemporaryDirectory() as path: + _repo_create(path) + + # A single run can be selected while debugging, which leaves the rest listed but not run + path_unit, file_list = _build(path, "common/error", test=2, tz="UTC", scale=4, log_timestamp=False) + test_c = _read(path, path_unit, "test.c") + + assert_in(" hrnAdd( 1, false);\n hrnAdd( 2, true);\n", test_c) + + # The timezone the test runs in, the scale of a performance test, and whether the log is timed + assert_in('hrnTzSet("UTC");\n', test_c) + assert_in("#define TEST_SCALE 4\n", test_c) + assert_in("#define TEST_TIMING false\n", test_c) + + # The log level the test itself runs at + path_unit, file_list = _build(path, "common/error", log_level_test=DETAIL) + + assert_in("#define LOG_LEVEL_TEST logLevelDetail\n", _read(path, path_unit, "test.c")) + + +#################################################################################################################################### +def test_build_clean(): + """Anything left in the unit path that this test did not generate is removed, since the path is reused.""" + + with tempfile.TemporaryDirectory() as path: + _repo_create(path) + + # Build one module and then another in the same path + _build(path, "common/exec") + path_unit, file_list = _build(path, "common/error") + + # What the first test generated is gone + assert_equal(file_list, ["meson.build", "meson_options.txt", "test.c"]) + + # What the build path holds is left alone, since that is ninja's and rebuilding it is what the path is reused to avoid + path_build = os.path.join(path_unit, "build") + + os.makedirs(path_build, exist_ok=True) + + with open(os.path.join(path_build, "build.ninja"), "w") as file: + file.write("") + + path_unit, file_list = _build(path, "common/error") + + assert_equal(file_list, ["build/build.ninja", "meson.build", "meson_options.txt", "test.c"]) + + +#################################################################################################################################### +def test_build_path(): + """A code module is mapped to where it lives, and a path is expressed relative to another.""" + + assert_equal(path_module("common/error"), "src/common/error") + assert_equal(path_module("test/common/harnessLog"), "test/src/common/harnessLog") + assert_equal(path_module("doc/command/build/build"), "doc/src/command/build/build") + + # Up out of the base path and back down into the compare path + assert_equal(path_relative("/test/unit-0/none", "/test/repo"), "../../repo") + assert_equal(path_relative("/test/unit-0/none", "/repo"), "../../../repo") + assert_equal(path_relative("/test", "/test/repo"), "repo") + + # A path relative to itself is nothing, which is a mistake rather than an empty path + with assert_raises(TestError) as error: + path_relative("/test/repo", "/test/repo") + + assert_equal(str(error.exception), "base and compare paths may not be equal") + + +#################################################################################################################################### +def test_build_shim_error(): + """Source the shim cannot make sense of is an error rather than a file that will not compile.""" + + # The shim walks back to the line above the function, so a function cannot be the first thing in the file + with assert_raises(TestError) as error: + build_shim("execOne(void)\n{\n}\n", ["execOne"]) + + assert_equal(str(error.exception), "shimmed function may not be on the first line") + + # The shim reads to the opening brace, which a properly formatted C file always has + with assert_raises(TestError) as error: + build_shim("void\nexecOne(\n void)\n", ["execOne"]) + + assert_equal(str(error.exception), "unable to find the end of the function signature") + + # A file that does not end with a linefeed would lose its last line + with assert_raises(TestError) as error: + build_shim("void\nexecOne(void)\n{\n}", ["execOne"]) + + assert_equal(str(error.exception), "shim module must end with a linefeed") + + # A name that only looks like the function is left alone, i.e. one that is longer or is not where a definition would be + assert_equal(build_shim("void\nexecOneMore(void)\n{\n}\n", ["execOne"]), "void\nexecOneMore(void)\n{\n}\n") + assert_equal(build_shim("void\n execOne(void)\n{\n}\n", ["execOne"]), "void\n execOne(void)\n{\n}\n") + + +#################################################################################################################################### +def test_build_user(): + """The user and group the test runs as are looked up once, since they are used several times and never change.""" + + # Whoever the tests run as, which is only known here by asking twice and getting the same answer + assert_equal(user_name(), user_name()) + assert_equal(group_name(), group_name()) + + assert_true(len(user_name()) > 0) + assert_true(len(group_name()) > 0) diff --git a/test/src/module/test/command/test/define_test.py b/test/src/module/test/command/test/define_test.py new file mode 100644 index 000000000..ffa7465ea --- /dev/null +++ b/test/src/module/test/command/test/define_test.py @@ -0,0 +1,331 @@ +"""Test Parse Define Yaml.""" + +#################################################################################################################################### +import os +import tempfile + +from harness.test import * + +from command.test.define import * +from common.error import * + +# A define file that exercises every shape the parser accepts. It is deliberately small since what matters here is the shapes rather +# than the test list, which is what the real file provides. +DEFINE = """ +unit: + - name: common + + test: + - name: error + total: 2 + + coverage: + - common/error + - common/errorInternal: noCode + - common/errorType: included + + - name: stack-trace + total: 1 + binReq: true + containerReq: true + define: -DDEBUG_TEST_TRACE + feature: stack-trace + harness: error + + include: + - common/stackTrace + + depend: + - common/error + - common/type/string + + - name: type-string + total: 3 + vm: [none, u22] + + include: + - common/error + + coverage: + - common/type/string + + harness: + - name: config + integration: false + + shim: + - common/config + - name: common/exec + function: + - execOne + - execTwo: + inc: common/execInc + + - name: test + lang: python + vm: [rh9] + + test: + - name: common/error + + coverage: + - test/common/error + + - name: common/log + lang: python + + coverage: + - test/common/log + +integration: + - name: real + db: true + + test: + - name: all + total: 1 + +performance: + - name: performance + + test: + - name: type + total: 1 +""" + +# A define file holding a single unit test, which is all the test error cases need +DEFINE_ERROR = """ +unit: + - name: common + + test: + - name: error +%s + +integration: [] +performance: [] +""" + +# A define file with a keyword that belongs to a test written at the module level instead +DEFINE_MODULE_ERROR = """ +unit: + - name: common + total: 1 + + test: + - name: error + +integration: [] +performance: [] +""" + + +#################################################################################################################################### +def _def_parse(define): + """Parse define yaml from a string by writing it where the parser looks for it.""" + + with tempfile.TemporaryDirectory() as path: + os.mkdir(os.path.join(path, "test")) + + with open(os.path.join(path, "test/define.yaml"), "w") as file: + file.write(define) + + return test_def_parse(path) + + +#################################################################################################################################### +def _def_parse_error(test_line_list): + """Parse a define file holding a single test built from the lines given, indented to where a test declares its keys.""" + + return _def_parse(DEFINE_ERROR % "\n".join(" " + line for line in test_line_list)) + + +#################################################################################################################################### +def test_define_module(): + """Every test becomes a module, in the order the file declares them.""" + + module_list = _def_parse(DEFINE) + + assert_equal( + [module.name for module in module_list], + [ + "common/error", + "common/stack-trace", + "common/type-string", + "test/common/error", + "test/common/log", + "real/all", + "performance/type", + ], + ) + + module = test_def_find(module_list, "common/stack-trace") + + assert_equal(module.type, TEST_TYPE_UNIT) + assert_equal(module.total, 1) + assert_true(module.bin_required) + assert_true(module.container_required) + assert_equal(module.flag, "-DDEBUG_TEST_TRACE") + assert_equal(module.include_list, ["common/stackTrace"]) + + # A feature is upper-cased and is available to the tests after the one that introduces it + assert_equal(module.feature, "STACK-TRACE") + assert_equal(module.feature_list, []) + assert_equal(test_def_find(module_list, "common/type-string").feature_list, ["STACK-TRACE"]) + + # PostgreSQL is required only by an integration module that declares it + assert_false(module.pg_required) + assert_true(test_def_find(module_list, "real/all").pg_required) + + # The type comes from the section the module was declared in + assert_equal(test_def_find(module_list, "real/all").type, TEST_TYPE_INTEGRATION) + assert_equal(test_def_find(module_list, "performance/type").type, TEST_TYPE_PERFORMANCE) + + +#################################################################################################################################### +def test_define_lang_vm(): + """Language and vm may be declared for a whole module and overridden by a test.""" + + module_list = _def_parse(DEFINE) + + # C is the default, and no vm means every vm + assert_equal(test_def_find(module_list, "common/error").lang, TEST_LANG_C) + assert_equal(test_def_find(module_list, "common/error").vm_list, []) + + # Declared for the whole module + assert_equal(test_def_find(module_list, "test/common/error").lang, TEST_LANG_PYTHON) + assert_equal(test_def_find(module_list, "test/common/error").vm_list, ["rh9"]) + + # Declared by the test itself + assert_equal(test_def_find(module_list, "test/common/log").lang, TEST_LANG_PYTHON) + assert_equal(test_def_find(module_list, "common/type-string").vm_list, ["none", "u22"]) + + +#################################################################################################################################### +def test_define_coverage(): + """A coverage entry is a code module, optionally tagged as included in another module or as having no code.""" + + module = test_def_find(_def_parse(DEFINE), "common/error") + + assert_equal([coverage.name for coverage in module.coverage_list], ["common/error", "common/errorInternal", "common/errorType"]) + + # A bare name is coverable and compiled on its own + assert_true(module.coverage_find("common/error").coverable) + assert_false(module.coverage_find("common/error").included) + + # noCode has nothing to cover, e.g. a header of macros + assert_false(module.coverage_find("common/errorInternal").coverable) + assert_true(module.coverage_find("common/errorInternal").included) + + # included is covered but compiled as part of another module + assert_true(module.coverage_find("common/errorType").coverable) + assert_true(module.coverage_find("common/errorType").included) + + # A module that is not covered is reported by returning nothing + assert_is_none(module.coverage_find("common/log")) + + +#################################################################################################################################### +def test_define_depend(): + """Dependencies accumulate in declaration order and each language accumulates its own.""" + + module_list = _def_parse(DEFINE) + + # The first test depends on nothing since it covers everything it declares + assert_equal(test_def_find(module_list, "common/error").depend_list, []) + + # What the tests before it covered, plus what it declares. A module already accumulated is not added again, which is why a test + # may safely declare a dependency that an earlier test has covered. + assert_equal(test_def_find(module_list, "common/stack-trace").depend_list, ["common/error", "common/type/string"]) + + # Minus what it covers or includes itself, since those are compiled into the test rather than depended on + assert_equal(test_def_find(module_list, "common/type-string").depend_list, []) + + # A python test accumulates only python modules, since it cannot import a C one + assert_equal(test_def_find(module_list, "test/common/error").depend_list, []) + assert_equal(test_def_find(module_list, "test/common/log").depend_list, ["test/common/error"]) + + # Accumulation carries on into the integration and performance tests + assert_equal(test_def_find(module_list, "real/all").depend_list, ["common/error", "common/type/string"]) + + +#################################################################################################################################### +def test_define_harness(): + """Harnesses accumulate across tests and carry the modules they shim.""" + + module_list = _def_parse(DEFINE) + + # A test declared before the harness does not have it + assert_equal(test_def_find(module_list, "common/error").harness_list, []) + + # A bare name is a harness that includes nothing + module = test_def_find(module_list, "common/stack-trace") + + assert_equal([harness.name for harness in module.harness_list], ["error"]) + assert_true(module.harness_list[0].integration) + assert_equal(module.harness_list[0].include_list, []) + assert_equal(module.shim_list, []) + + # A harness that shims modules includes them in the order they are listed + module = test_def_find(module_list, "common/type-string") + harness = module.harness_list[1] + + assert_equal([harness.name for harness in module.harness_list], ["error", "config"]) + assert_false(harness.integration) + assert_equal(harness.include_list, ["common/config", "common/exec"]) + + +#################################################################################################################################### +def test_define_shim(): + """A shim names the functions the harness replaces, which may be defined in a module included by the shim.""" + + module = test_def_find(_def_parse(DEFINE), "common/type-string") + + # A shimmed module with no functions is included in the harness but has nothing to shim + assert_is_none(module.shim_find("common/config")) + + shim = module.shim_find("common/exec") + + assert_equal(shim.name, "common/exec") + + # The shim takes the integration flag of the harness that declared it + assert_false(shim.integration) + + # A bare name is defined in the shim module, else the included module it is defined in is named + assert_equal([function.name for function in shim.function_list], ["execOne", "execTwo"]) + assert_is_none(shim.function_list[0].inc) + assert_equal(shim.function_list[1].inc, "common/execInc") + + +#################################################################################################################################### +def test_define_error(): + """A definition the parser does not accept is an error that says what was wrong.""" + + # A key that belongs to a test, written at the module level where it would be silently ignored + with assert_raises(TestError) as error: + _def_parse(DEFINE_MODULE_ERROR) + + assert_equal(str(error.exception), "unexpected keyword 'total' in module 'common'") + + # Everything else is declared by a test + for line_list, message in ( + (["bogus: true"], "unexpected keyword 'bogus' in test 'common/error'"), + (["lang: rust"], "invalid lang 'rust' in test 'common/error'"), + (["coverage:", " - common/error: bogus"], "invalid coverage type bogus"), + (["harness:", " - name: config", " bogus: true"], "invalid key 'bogus'"), + ( + ["harness:", " - name: config", " shim:", " - name: common/exec", " bogus: true"], + "invalid key 'bogus'", + ), + (["harness:", " - name: config", " shim:", " - function:", " - execOne"], "shim name is required"), + ): + with assert_raises(TestError) as error: + _def_parse_error(line_list) + + assert_equal(str(error.exception), message, line_list) + + # A test that is not in the file + with assert_raises(TestError) as error: + test_def_find(_def_parse(DEFINE), "bogus/test") + + assert_equal(str(error.exception), "'bogus/test' is not a valid test") diff --git a/test/src/module/test/command/test/python_test.py b/test/src/module/test/command/test/python_test.py new file mode 100644 index 000000000..132b4a572 --- /dev/null +++ b/test/src/module/test/command/test/python_test.py @@ -0,0 +1,201 @@ +"""Test Python Test Runner. + +The runner is what loads and runs a test module, so testing it means running a test module from inside one. The module it runs is +written to a temporary path and the interpreter state the runner changes -- the import path, the meta path, and the coverage +measurement -- is put back afterwards, since this interpreter carries on running the tests after the runner returns. + +The coverage measurement is replaced rather than started for real, since a second measurement in the same interpreter would take +over tracing from the one already measuring this test.""" + +#################################################################################################################################### +import io +import os +import sys +import tempfile +from contextlib import redirect_stdout +from unittest.mock import patch + +from harness.test import * + +from command.test.python import * + +# A test module with one test that passes and one that fails, named so either can be selected +TEST_MODULE = """ +def test_pass(): + pass + + +def test_fail(): + raise Exception("this test fails") +""" + + +#################################################################################################################################### +def _config(path_lib, path_test, allow="", name="", coverage=None): + """Build what the runner reads from the command line, which is what cfg_parse returns.""" + + return cfg_parse( + ["--lib=%s" % path_lib, "--test=%s" % path_test, "--allow=%s" % allow, "--name=%s" % name] + + ([] if coverage is None else ["--coverage=%s" % coverage]) + ) + + +#################################################################################################################################### +def _run(config): + """Run a test module, returning the status and what it wrote. + + The import path and meta path are put back since the runner adds to both and never takes them off, which is right for a process + that exits when the module is done and wrong for this one.""" + + output = io.StringIO() + path_list = list(sys.path) + meta_path_list = list(sys.meta_path) + + try: + with redirect_stdout(output): + result = test_module_run(config) + finally: + sys.path[:] = path_list + sys.meta_path[:] = meta_path_list + + return result, output.getvalue() + + +#################################################################################################################################### +def _module_write(path, content=TEST_MODULE): + """Write a test module and return its path.""" + + result = os.path.join(path, "module_test.py") + + with open(result, "w") as file: + file.write(content) + + return result + + +#################################################################################################################################### +def test_python_run(): + """A module runs every test in it and reports failure when any of them fail.""" + + path_lib = os.path.dirname(os.path.dirname(os.path.abspath(sys.modules["harness.test"].__file__))) + + with tempfile.TemporaryDirectory() as path: + path_test = _module_write(path) + + # A module with a failing test fails + result, output = _run(_config(path_lib, path_test)) + + assert_equal(result, 1) + assert_in("test_pass ... ok", output) + assert_in("test_fail ... ERROR", output) + + # A single test can be selected, which is how one test is run while debugging + result, output = _run(_config(path_lib, path_test, name="test_pass")) + + assert_equal(result, 0) + assert_in("Ran 1 test", output) + assert_not_in("test_fail", output) + + +#################################################################################################################################### +def test_python_guard(): + """A library module the test did not declare cannot be imported.""" + + path_lib = os.path.dirname(os.path.dirname(os.path.abspath(sys.modules["harness.test"].__file__))) + + with tempfile.TemporaryDirectory() as path: + # A module that was not declared is refused, with what to do about it. The import is in the test rather than at the top of + # the module so the guard reports it as a failure rather than a load error. This runs before the case below because an + # import that succeeded is cached in sys.modules and never reaches the guard again, which is the whole reason the runner + # is a separate interpreter. + path_test = _module_write(path, "def test_import():\n import common.render\n") + + result, output = _run(_config(path_lib, path_test)) + + assert_equal(result, 1) + assert_in("'common/render' is not declared by this test module", output) + assert_in("add it to coverage or depend in define.yaml", output) + + # A module that was declared, a package on the way to it, and something outside the library are all allowed. What the + # declared module imports has to be declared as well, all the way down, since the guard sees every import rather than only + # the ones the test module makes itself. + path_test = _module_write( + path, + "def test_import():\n" " import os\n" " import common\n" " import common.render\n", + ) + + result, output = _run(_config(path_lib, path_test, allow="common.render,common.string_id,common.error")) + + assert_equal(result, 0) + + +#################################################################################################################################### +def test_python_coverage(): + """Coverage is measured when it is asked for and refused when it cannot be measured.""" + + path_lib = os.path.dirname(os.path.dirname(os.path.abspath(sys.modules["harness.test"].__file__))) + + with tempfile.TemporaryDirectory() as path: + path_test = _module_write(path, "def test_pass():\n pass\n") + path_coverage = os.path.join(path, "coverage.json") + + # The measurement is replaced so this test does not take tracing away from the measurement already running + with patch("coverage.Coverage") as measure: + result, output = _run(_config(path_lib, path_test, coverage=path_coverage)) + + assert_equal(result, 0) + + # Measuring starts before the module is loaded and the report is written where it was asked for + measure.return_value.start.assert_called_once_with() + measure.return_value.stop.assert_called_once_with() + measure.return_value.json_report.assert_called_once_with(outfile=path_coverage) + + # A coverage too old to report branch detail is refused, since a summary cannot say which line to fix + with patch("coverage.__version__", "6.4.4"): + result, output = _run(_config(path_lib, path_test, coverage=path_coverage)) + + assert_equal(result, 2) + assert_in("coverage 6.4.4 is too old, 6.5 or newer is required for branch detail", output) + + # Coverage that is not installed at all is refused with what to install + with patch.dict(sys.modules, {"coverage": None}): + result, output = _run(_config(path_lib, path_test, coverage=path_coverage)) + + assert_equal(result, 2) + assert_in("unable to measure coverage: no module named 'coverage'", output) + assert_in("HINT: install python3-coverage", output) + + +#################################################################################################################################### +def test_python_main(): + """The command line is what the harness passes, and main is only the wiring between it and the run.""" + + path_lib = os.path.dirname(os.path.dirname(os.path.abspath(sys.modules["harness.test"].__file__))) + + with tempfile.TemporaryDirectory() as path: + path_test = _module_write(path, "def test_pass():\n pass\n") + + config = _config(path_lib, path_test, allow="common.render,common.log", name="test_pass") + + assert_equal(config.lib, path_lib) + assert_equal(config.test, path_test) + assert_equal(config.allow, "common.render,common.log") + assert_equal(config.name, "test_pass") + assert_is_none(config.coverage) + + # Everything main does beyond parsing is already set here, so running it again changes nothing + output = io.StringIO() + path_list = list(sys.path) + meta_path_list = list(sys.meta_path) + + try: + with patch.object(sys, "argv", ["python.py", "--lib=%s" % path_lib, "--test=%s" % path_test]): + with redirect_stdout(output): + result = main() + finally: + sys.path[:] = path_list + sys.meta_path[:] = meta_path_list + + assert_equal(result, 0) + assert_true(sys.dont_write_bytecode) + assert_in("Ran 1 test", output.getvalue()) diff --git a/test/src/module/test/command/test/unit_test.py b/test/src/module/test/command/test/unit_test.py new file mode 100644 index 000000000..06e0bfa84 --- /dev/null +++ b/test/src/module/test/command/test/unit_test.py @@ -0,0 +1,383 @@ +"""Test Unit Command. + +The build and the commands it runs are replaced, since what this module does is decide what to build and what to run rather than +build or run it. Everything is driven through the command itself rather than reaching for the helpers it is made of, which is how +the C tests are written.""" + +#################################################################################################################################### +import io +import os +import tempfile +from contextlib import redirect_stdout +from unittest.mock import patch + +from harness.test import * + +from command.test.unit import * +from common.error import * +from common.log import * + +# A define file with the C and python tests the command has to tell apart +DEFINE = """ +unit: + - name: common + + test: + - name: error + total: 1 + + coverage: + - common/error + + - name: release + total: 1 + define: -DNDEBUG + + coverage: + - common/release + + - name: bogus + total: 1 + define: -DBOGUS + + coverage: + - common/bogus + + - name: test + lang: python + + test: + - name: common/log + + coverage: + - test/common/log + - test/common/logInternal: noCode + + depend: + - test/common/error + + include: + - test/common/render + +integration: [] +performance: + - name: performance + + test: + - name: type + total: 1 + + coverage: + - performance/type +""" + + +#################################################################################################################################### +class Config: + """What the unit command reads from the command line.""" + + def __init__(self, repo_path, test_path, module, **option): + self.repo_path = repo_path + self.test_path = test_path + self.module = module + self.vm = "none" + self.vm_id = 0 + self.vm_arch = None + self.pg_version = "invalid" + self.test = None + self.test_name = None + self.coverage_file = None + self.scale = 1 + self.tz = None + self.log_level_test = OFF + self.log_timestamp = True + self.coverage = True + self.back_trace = True + self.optimize = False + self.profile = False + + for name, value in option.items(): + setattr(self, name, value) + + +#################################################################################################################################### +def _repo_create(path, define=DEFINE, test_module=("test/common/log",)): + """Write the repository the command reads, i.e. the define file and the python test modules it names.""" + + result = os.path.join(path, "repo") + + os.makedirs(os.path.join(result, "test/src/module/test/common"), exist_ok=True) + + with open(os.path.join(result, "test/define.yaml"), "w") as file: + file.write(define) + + for name in test_module: + with open(os.path.join(result, "test/src/module", name + "_test.py"), "w") as file: + file.write("def test_pass():\n pass\n") + + return result + + +#################################################################################################################################### +def _cmd_unit(config, exec_result=None): + """Run the unit command with the build and the commands it runs replaced. + + Returns the commands that would have been run, what was written, and the error raised, if any. The exec results are returned in + order, and an exception in the list is raised instead.""" + + command_list = [] + output = io.StringIO() + error = None + + def exec_fake(command, result_expect=0): + command_list.append(command) + result = exec_result.pop(0) if exec_result else "" + + if isinstance(result, Exception): + raise result + + return result + + log_init(WARN, False) + + try: + with patch("command.test.unit.exec_one", exec_fake), patch("command.test.unit.TestBuild") as build: + build.return_value.path_unit = os.path.join(config.test_path, "unit-0/none") + build.return_value.path_unit_build = os.path.join(config.test_path, "unit-0/none/build") + + try: + with redirect_stdout(output): + cmd_unit(config) + except TestError as exception: + error = str(exception) + + _cmd_unit.build = build + finally: + log_init(INFO, True) + + return command_list, output.getvalue(), error + + +#################################################################################################################################### +def test_unit_python(): + """A python module has nothing to build so it is run where it is, with only what it declared importable.""" + + with tempfile.TemporaryDirectory() as path: + path_repo = _repo_create(path) + + command_list, output, error = _cmd_unit(Config(path_repo, path, "test/common/log"), ["ran the tests\n"]) + + assert_is_none(error) + + # The output comes back as it was written rather than through the log, which would indent it as a continuation + assert_equal(output, "ran the tests\n") + + # The runner is given the code modules the test covers, what it declared, and what earlier tests covered. A module with no + # code is not importable so it is not in the list. + assert_equal(len(command_list), 1) + assert_in("--allow='common.log,common.error,common.render'", command_list[0]) + + # There is no coverage file so coverage is not measured, and no test name so every test runs + assert_in("--name=''", command_list[0]) + assert_not_in("--coverage=", command_list[0]) + + +#################################################################################################################################### +def test_unit_python_option(): + """A single test can be selected and coverage is written where the caller asks for it.""" + + with tempfile.TemporaryDirectory() as path: + path_repo = _repo_create(path) + path_coverage = os.path.join(path, "result/coverage/raw/test-log.json") + + command_list, output, error = _cmd_unit( + Config(path_repo, path, "test/common/log", test_name="test_pass", coverage_file=path_coverage) + ) + + assert_is_none(error) + assert_in("--name='test_pass'", command_list[0]) + assert_in("--coverage='%s'" % path_coverage, command_list[0]) + + # The path is created since the harness runs from a copy of the repository and the report is built from the original + assert_true(os.path.isdir(os.path.dirname(path_coverage))) + + # Coverage is not measured when the build is not measuring it, even when a file was named + command_list, output, error = _cmd_unit( + Config(path_repo, path, "test/common/log", coverage=False, coverage_file=path_coverage) + ) + + assert_not_in("--coverage=", command_list[0]) + + +#################################################################################################################################### +def test_unit_python_error(): + """A python test module that is not there is an error that names the file that was expected.""" + + with tempfile.TemporaryDirectory() as path: + path_repo = _repo_create(path, test_module=()) + + command_list, output, error = _cmd_unit(Config(path_repo, path, "test/common/log")) + + assert_equal(error, "unable to find test module '%s/test/src/module/test/common/log_test.py'" % path_repo) + assert_equal(command_list, []) + + +#################################################################################################################################### +def test_unit_build(): + """A C module is set up and built, with the architecture taken from the machine when it was not given.""" + + with tempfile.TemporaryDirectory() as path: + path_repo = _repo_create(path) + + command_list, output, error = _cmd_unit(Config(path_repo, path, "common/error"), ["x86_64\n"]) + + assert_is_none(error) + + # The architecture comes from the machine, then meson is set up since there is nothing built yet, then ninja builds it + assert_equal(command_list[0], "uname -m") + assert_in("meson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug -Db_coverage=true", command_list[1]) + assert_in("ninja -C '%s/unit-0/none/build'" % path, command_list[2]) + + # The architecture is passed to the build + assert_equal(_cmd_unit.build.call_args[0][2], "x86_64") + + # A machine that reports an architecture under another name is normalized to the name the project uses + for report, expect in (("i686\n", "i386"), ("arm64\n", "aarch64"), ("aarch64\n", "aarch64")): + _cmd_unit(Config(path_repo, path, "common/error"), [report]) + + assert_equal(_cmd_unit.build.call_args[0][2], expect, report) + + # An architecture that was given is used as it is, so the machine is not asked + command_list, output, error = _cmd_unit(Config(path_repo, path, "common/error", vm_arch="ppc64le")) + + assert_equal(_cmd_unit.build.call_args[0][2], "ppc64le") + assert_not_in("uname -m", command_list) + + +#################################################################################################################################### +def test_unit_build_type(): + """The build type is release when the module cannot be built with debug, else debug.""" + + with tempfile.TemporaryDirectory() as path: + path_repo = _repo_create(path) + + # A module that sets a define + command_list, output, error = _cmd_unit(Config(path_repo, path, "common/release", vm_arch="x86_64")) + + assert_is_none(error) + assert_in("-Dbuildtype=release", command_list[0]) + + # A performance test, which is timed so it cannot be built with debug either + command_list, output, error = _cmd_unit(Config(path_repo, path, "performance/type", vm_arch="x86_64")) + + assert_in("-Dbuildtype=release", command_list[0]) + + # Profiling, which is also release, and turns coverage off since the two cannot both be measured + command_list, output, error = _cmd_unit( + Config(path_repo, path, "common/error", vm_arch="x86_64", profile=True, coverage=False) + ) + + assert_in("-Dbuildtype=release -Db_coverage=false", command_list[0]) + + # A define other than the one that turns debug off is not something the build knows what to do with + command_list, output, error = _cmd_unit(Config(path_repo, path, "common/bogus", vm_arch="x86_64")) + + assert_equal(error, "unexpected define '-DBOGUS'") + + +#################################################################################################################################### +def test_unit_build_reuse(): + """A unit path that was already set up is reconfigured rather than set up again, and stale data is cleared out.""" + + with tempfile.TemporaryDirectory() as path: + path_repo = _repo_create(path) + path_build = os.path.join(path, "unit-0/none/build") + path_object = os.path.join(path_build, "test-unit.p") + + os.makedirs(path_object) + + for name in ("build.ninja", "gmon.out"): + with open(os.path.join(path_build, name), "w") as file: + file.write("") + + # Coverage and profile data from the last run, which would otherwise be counted again + for name in ("stale.gcda", "stale.gcno"): + with open(os.path.join(path_object, name), "w") as file: + file.write("") + + command_list, output, error = _cmd_unit(Config(path_repo, path, "common/error", vm_arch="x86_64")) + + assert_is_none(error) + assert_in("meson configure -Dbuildtype=debug -Db_coverage=true", command_list[0]) + + # The coverage data is gone but what the build needs to link is left alone + assert_false(os.path.exists(os.path.join(path_object, "stale.gcda"))) + assert_true(os.path.exists(os.path.join(path_object, "stale.gcno"))) + assert_false(os.path.exists(os.path.join(path_build, "gmon.out"))) + + +#################################################################################################################################### +def test_unit_build_retry(): + """A build that fails is retried once from a clean path, since a stale build path is the usual cause.""" + + with tempfile.TemporaryDirectory() as path: + path_repo = _repo_create(path) + path_unit = os.path.join(path, "unit-0/none") + path_stale = os.path.join(path_unit, "stale") + + os.makedirs(path_stale) + + # The first ninja fails and the retry succeeds + command_list, output, error = _cmd_unit( + Config(path_repo, path, "common/error", vm_arch="x86_64"), ["", TestError("ninja said no"), "", ""] + ) + + assert_is_none(error) + assert_in("build failed for unit common/error -- will retry: ninja said no", output) + + # The unit path was emptied before the retry + assert_false(os.path.exists(path_stale)) + assert_true(os.path.isdir(path_unit)) + + # A second failure is the end of it, with the error from the build rather than from the retry + command_list, output, error = _cmd_unit( + Config(path_repo, path, "common/error", vm_arch="x86_64"), + ["", TestError("ninja said no"), "", TestError("ninja said no again")], + ) + + assert_equal(error, "build failed for unit common/error: ninja said no again") + + +#################################################################################################################################### +def test_unit_build_retry_clean(): + """A unit path that was never created is nothing to empty, which is the first build on a clean test path.""" + + with tempfile.TemporaryDirectory() as path: + path_repo = _repo_create(path) + path_unit = os.path.join(path, "unit-0/none") + + assert_false(os.path.exists(path_unit)) + + command_list, output, error = _cmd_unit( + Config(path_repo, path, "common/error", vm_arch="x86_64"), ["", TestError("ninja said no"), "", ""] + ) + + assert_is_none(error) + assert_true(os.path.isdir(path_unit)) + + +#################################################################################################################################### +def test_unit_build_retry_mode(): + """A unit path a test left unreadable is reset before it is emptied, since a test may run as another user.""" + + with tempfile.TemporaryDirectory() as path: + path_repo = _repo_create(path) + + # The path cannot be removed until the mode is reset, which is the second thing tried + with patch("command.test.unit.shutil.rmtree", side_effect=[OSError("permission denied"), None]): + command_list, output, error = _cmd_unit( + Config(path_repo, path, "common/error", vm_arch="x86_64"), ["", TestError("ninja said no"), "", ""] + ) + + assert_is_none(error) + assert_in("chmod -R 777 '%s/unit-0/none'" % path, command_list) diff --git a/test/src/module/test/common/error_test.py b/test/src/module/test/common/error_test.py new file mode 100644 index 000000000..672ed9aa9 --- /dev/null +++ b/test/src/module/test/common/error_test.py @@ -0,0 +1,52 @@ +"""Test Error Handling.""" + +#################################################################################################################################### +from harness.test import * + +from common.error import * + + +#################################################################################################################################### +def test_error_message(): + """The message is what gets reported and the status is what the harness exits with.""" + + error = TestError("something went wrong") + + assert_equal(str(error), "something went wrong") + assert_equal(error.status, EXIT_ERROR) + + # Ordinary handlers must be able to catch it + assert_is_instance(error, Exception) + + +#################################################################################################################################### +def test_error_status(): + """A status other than the default is kept. + + That is how the coverage command reports incomplete coverage rather than a failure.""" + + assert_equal(TestError("incomplete", 1).status, 1) + assert_equal(TestError("failed", 0).status, 0) + + # The default is distinct from the incomplete coverage status so the two are never confused + assert_not_equal(EXIT_ERROR, 1) + + +#################################################################################################################################### +def test_check_pass(): + """A condition that holds passes silently.""" + + for condition in (True, 1, "text", [0], {"key": "value"}): + assert_is_none(check(condition, "should not raise"), condition) + + +#################################################################################################################################### +def test_check_fail(): + """A condition that does not hold raises with the message given.""" + + for condition in (False, 0, "", [], {}, None): + with assert_raises(TestError) as error: + check(condition, "invariant does not hold") + + assert_equal(str(error.exception), "invariant does not hold") + assert_equal(error.exception.status, EXIT_ERROR) diff --git a/test/src/module/test/common/exec_test.py b/test/src/module/test/common/exec_test.py new file mode 100644 index 000000000..e9c318d04 --- /dev/null +++ b/test/src/module/test/common/exec_test.py @@ -0,0 +1,48 @@ +"""Test Execute Process.""" + +#################################################################################################################################### +from harness.test import * + +from common.error import * +from common.exec import * + + +#################################################################################################################################### +def test_exec_output(): + """The combined output is returned on success.""" + + assert_equal(exec_one("echo hello"), "hello\n") + + # The command runs through the shell so shell syntax works + assert_equal(exec_one("echo hello | tr a-z A-Z"), "HELLO\n") + + # Output written to stderr is folded in, which is where meson and ninja report what went wrong + assert_equal(exec_one("echo out; echo err 1>&2"), "out\nerr\n") + + +#################################################################################################################################### +def test_exec_error(): + """An unexpected exit status is an error that carries the output.""" + + with assert_raises(TestError) as error: + exec_one("echo bad && false") + + assert_equal(str(error.exception), "echo bad && false terminated unexpectedly [1]: bad") + + # A command that wrote nothing reports only the status + with assert_raises(TestError) as error: + exec_one("exit 7") + + assert_equal(str(error.exception), "exit 7 terminated unexpectedly [7]") + + +#################################################################################################################################### +def test_exec_expect(): + """A status other than zero can be the expected one, which then makes zero unexpected.""" + + assert_equal(exec_one("exit 3", 3), "") + + with assert_raises(TestError) as error: + exec_one("true", 3) + + assert_equal(str(error.exception), "true terminated unexpectedly [0]") diff --git a/test/src/module/test/common/log_test.py b/test/src/module/test/common/log_test.py new file mode 100644 index 000000000..64482e947 --- /dev/null +++ b/test/src/module/test/common/log_test.py @@ -0,0 +1,112 @@ +"""Test Console Logging.""" + +#################################################################################################################################### +import io +import re +from contextlib import redirect_stdout + +from harness.test import * + +from common.log import * + + +#################################################################################################################################### +def _capture(function, level=INFO, timestamp=False): + """Run a function with the log configured as specified and return what it wrote. + + Timestamps are off unless a test asks for them so the output is exact. The settings are module state shared by every test so + they are put back afterwards.""" + + output = io.StringIO() + + log_init(level, timestamp) + + try: + with redirect_stdout(output): + function() + finally: + log_init(INFO, True) + + return output.getvalue() + + +#################################################################################################################################### +def test_log_level_parse(): + """A level name is converted to its id and an unknown name is not an error.""" + + assert_equal(log_level_parse("off"), OFF) + assert_equal(log_level_parse("error"), ERROR) + assert_equal(log_level_parse("trace"), TRACE) + + # An unknown name is reported by returning nothing, since the caller has better context for the error + assert_is_none(log_level_parse("bogus")) + assert_is_none(log_level_parse("")) + + # Every level can be parsed back from the name it is logged with + for level, name in LEVEL_NAME.items(): + assert_equal(log_level_parse(name), level, name) + + assert_equal(len(LEVEL_ID), len(LEVEL_NAME)) + + # The values match LogLevel in src/common/logLevel.h, which the enum below is built from + assert_equal((OFF, ERROR, WARN, INFO, DETAIL, DEBUG, TRACE), (0, 2, 3, 4, 5, 6, 7)) + + +#################################################################################################################################### +def test_log_level_enum(): + """The enum name matches LogLevel in src/common/logLevel.h, which the generated test.c is substituted with.""" + + assert_equal(log_level_enum(OFF), "logLevelOff") + assert_equal(log_level_enum(INFO), "logLevelInfo") + assert_equal(log_level_enum(DETAIL), "logLevelDetail") + + +#################################################################################################################################### +def test_log_format(): + """The level is right-aligned in the prefix and continuation lines are indented under it.""" + + assert_equal(_capture(lambda: log(INFO, "message")), "P00 INFO: message\n") + assert_equal(_capture(lambda: log(DETAIL, "message"), DETAIL), "P00 DETAIL: message\n") + + # Lines after the first line up under the first + assert_equal(_capture(lambda: log(INFO, "first\nsecond")), "P00 INFO: first\n" + " " * len("P00 INFO: ") + "second\n") + + # A message that is not a string is rendered as one + assert_equal(_capture(lambda: log(INFO, 404)), "P00 INFO: 404\n") + + +#################################################################################################################################### +def test_log_timestamp(): + """A timestamp is prepended unless it is suppressed.""" + + output = _capture(lambda: log(INFO, "message"), timestamp=True) + + assert_true(re.match(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} P00 INFO: message\n$", output), output) + + # Continuation lines are indented past the timestamp as well + line_list = _capture(lambda: log(INFO, "first\nsecond"), timestamp=True).split("\n") + + assert_equal(len(line_list[0]) - len("first"), len(line_list[1]) - len("second")) + + +#################################################################################################################################### +def test_log_level(): + """A message above the current level is not written.""" + + assert_equal(_capture(lambda: log(DEBUG, "message")), "") + assert_equal(_capture(lambda: log(DEBUG, "message"), DEBUG), "P00 DEBUG: message\n") + + # Off is below every level so nothing at all is written + assert_equal(_capture(lambda: log(ERROR, "message"), OFF), "") + + +#################################################################################################################################### +def test_log_write(): + """Every level writes under its own name.""" + + for level in (ERROR, WARN, INFO, DETAIL, DEBUG, TRACE): + assert_equal( + _capture(lambda: log(level, "message"), TRACE), + "P00 %6s: message\n" % LEVEL_NAME[level].upper(), + LEVEL_NAME[level], + ) diff --git a/test/src/module/test/common/render_test.py b/test/src/module/test/common/render_test.py new file mode 100644 index 000000000..b399b1292 --- /dev/null +++ b/test/src/module/test/common/render_test.py @@ -0,0 +1,49 @@ +"""Test Render Helpers. + +The StringId values expected here are taken from StringId macros in the project source, which the linter already checks against the +C encoder, so they are known good rather than whatever this implementation happens to produce.""" + +#################################################################################################################################### +from harness.test import * + +from common.render import * + + +#################################################################################################################################### +def test_render_enum(): + """A dashed value becomes camel case.""" + + assert_equal(bld_enum(None, "info"), "info") + assert_equal(bld_enum(None, "type-convert"), "typeConvert") + + # The module name of a test, which is what the generated test.c includes + assert_equal(bld_enum(None, "common/type-string"), "common/typeString") + + # Successive dashes and a trailing dash have nothing to upper-case + assert_equal(bld_enum(None, "a--b"), "aB") + assert_equal(bld_enum(None, "trail-"), "trail") + + +#################################################################################################################################### +def test_render_enum_prefix(): + """A prefix is prepended and upper-cases the first letter of the value.""" + + assert_equal(bld_enum("logLevel", "info"), "logLevelInfo") + assert_equal(bld_enum("logLevel", "off"), "logLevelOff") + + # An empty prefix is not the same as no prefix since it still upper-cases + assert_equal(bld_enum("", "info"), "Info") + + # An empty value leaves the prefix alone + assert_equal(bld_enum("prefix", ""), "prefix") + + +#################################################################################################################################### +def test_render_str_id(): + """The rendered macro is what the linter compares the source against.""" + + assert_equal(bld_str_id_seq("any"), 'STRID5("any", 0x65c10)') + assert_equal(bld_str_id_seq("lz4"), 'STRID6("lz4", 0x2068c1)') + assert_equal(bld_str_id_seq("asc", 1), 'STRID5S("asc", 1, 0xe614)') + assert_equal(bld_str_id_seq("time", 6), 'STRID5S("time", 6, 0x56a680e)') + assert_equal(bld_str_id_seq("execId"), 'STRID6("execId", 0x12e0c56051)') diff --git a/test/src/module/test/common/storage_test.py b/test/src/module/test/common/storage_test.py new file mode 100644 index 000000000..c25d5cf4f --- /dev/null +++ b/test/src/module/test/common/storage_test.py @@ -0,0 +1,187 @@ +"""Test File System Helpers. + +The error other than missing is provoked with a path where a file is expected, since that is reachable without changing permissions +and the tests must behave the same when run as root.""" + +#################################################################################################################################### +import os +import tempfile + +from harness.test import * + +from common.error import * +from common.storage import * + + +#################################################################################################################################### +def test_file_read_write(): + """A file that is written is read back and parent paths are created as needed.""" + + with tempfile.TemporaryDirectory() as path: + path_file = os.path.join(path, "sub/dir/file.txt") + + file_write(path_file, "content") + + assert_equal(file_read(path_file), "content") + assert_true(os.path.isdir(os.path.join(path, "sub/dir"))) + + # An existing file is replaced + file_write(path_file, "replaced") + + assert_equal(file_read(path_file), "replaced") + + +#################################################################################################################################### +def test_file_read_error(): + """A file that cannot be read is an error unless it is merely missing and that is allowed.""" + + with tempfile.TemporaryDirectory() as path: + path_file = os.path.join(path, "missing.txt") + + with assert_raises(TestError) as error: + file_read(path_file) + + assert_equal(str(error.exception), "unable to open file '%s' for read" % path_file) + + # Missing is reported by returning nothing when the caller expects it, e.g. a generated file that is not there yet + assert_is_none(file_read(path_file, ignore_missing=True)) + + # Anything else reports what the system said, and is an error however the caller asked for it + with assert_raises(TestError) as error: + file_read(path, ignore_missing=True) + + assert_equal(str(error.exception), "unable to open file '%s' for read: Is a directory" % path) + + +#################################################################################################################################### +def test_file_write_error(): + """A file that cannot be written reports what the system said.""" + + with tempfile.TemporaryDirectory() as path: + with assert_raises(TestError) as error: + file_write(path, "content") + + assert_equal(str(error.exception), "unable to open file '%s' for write: Is a directory" % path) + + +#################################################################################################################################### +def test_file_write_differs(): + """A file is written only when the content differs so an unchanged file does not trigger a rebuild.""" + + with tempfile.TemporaryDirectory() as path: + path_file = os.path.join(path, "file.txt") + + # Missing counts as different + file_write_differs(path_file, "content") + + assert_equal(file_read(path_file), "content") + + # Content that matches leaves the file alone, which only the timestamp can show since the content is the same either way + os.utime(path_file, (0, 0)) + file_write_differs(path_file, "content") + + assert_equal(os.stat(path_file).st_mtime, 0) + + # Content that differs is written + file_write_differs(path_file, "changed") + + assert_equal(file_read(path_file), "changed") + assert_not_equal(os.stat(path_file).st_mtime, 0) + + +#################################################################################################################################### +def test_file_remove(): + """A file is removed and missing is only an error when the caller asks for it.""" + + with tempfile.TemporaryDirectory() as path: + path_file = os.path.join(path, "file.txt") + + file_write(path_file, "content") + file_remove(path_file) + + assert_false(os.path.exists(path_file)) + + # Missing is not an error by default since a file that is gone is what the caller wanted + assert_is_none(file_remove(path_file)) + + with assert_raises(TestError) as error: + file_remove(path_file, error_on_missing=True) + + assert_equal(str(error.exception), "unable to remove missing file '%s'" % path_file) + + # Anything else reports what the system said + with assert_raises(TestError) as error: + file_remove(path) + + assert_equal(str(error.exception), "unable to remove file '%s': Is a directory" % path) + + +#################################################################################################################################### +def test_path_create(): + """Missing parents are created and an existing path is not an error.""" + + with tempfile.TemporaryDirectory() as path: + path_sub = os.path.join(path, "sub/dir") + + path_create(path_sub) + + assert_true(os.path.isdir(path_sub)) + assert_is_none(path_create(path_sub)) + + # An empty path is where creation stops, e.g. a file written with no path in the name + assert_is_none(path_create("")) + + # A path that cannot be created reports what the system said + path_file = os.path.join(path, "file.txt") + file_write(path_file, "content") + + with assert_raises(TestError) as error: + path_create(os.path.join(path_file, "sub")) + + assert_equal(str(error.exception), "unable to create path '%s': Not a directory" % os.path.join(path_file, "sub")) + + +#################################################################################################################################### +def test_path_list(): + """Names are listed sorted and can be filtered.""" + + with tempfile.TemporaryDirectory() as path: + for name in ("c.txt", "a.txt", "b.json"): + file_write(os.path.join(path, name), "content") + + path_create(os.path.join(path, "sub")) + + # Paths are listed along with files + assert_equal(path_list(path), ["a.txt", "b.json", "c.txt", "sub"]) + assert_equal(path_list(path, expression=r"\.txt$"), ["a.txt", "c.txt"]) + + # A missing path is empty rather than an error, since a path listed here is often legitimately absent + path_missing = os.path.join(path, "missing") + + assert_equal(path_list(path_missing), []) + + with assert_raises(TestError) as error: + path_list(path_missing, error_on_missing=True) + + assert_equal(str(error.exception), "unable to list missing path '%s'" % path_missing) + + # Anything else reports what the system said + path_file = os.path.join(path, "a.txt") + + with assert_raises(TestError) as error: + path_list(path_file) + + assert_equal(str(error.exception), "unable to list path '%s': Not a directory" % path_file) + + +#################################################################################################################################### +def test_path_list_recurse(): + """Files are listed recursively and relative to the path, with the paths themselves left out.""" + + with tempfile.TemporaryDirectory() as path: + file_write(os.path.join(path, "b.txt"), "content") + file_write(os.path.join(path, "sub/a.txt"), "content") + file_write(os.path.join(path, "sub/deep/c.txt"), "content") + path_create(os.path.join(path, "empty")) + + assert_equal(path_list_recurse(path), ["b.txt", "sub/a.txt", "sub/deep/c.txt"]) diff --git a/test/src/module/test/common/string_test.py b/test/src/module/test/common/string_test.py new file mode 100644 index 000000000..a93e7fea6 --- /dev/null +++ b/test/src/module/test/common/string_test.py @@ -0,0 +1,93 @@ +"""Test String Modules. + +The StringId values expected here are taken from StringId macros in the project source, which the linter already checks against the +C encoder, so they are known good rather than whatever this implementation happens to produce.""" + +#################################################################################################################################### +from harness.test import * + +from common.error import * +from common.string_id import * + + +#################################################################################################################################### +def test_string_id_encode_5bit(): + """5-bit encoding covers lower case, dash, and the digits 2, 5, and 6.""" + + assert_equal(str_id_from_z("any"), 0x65C10) + assert_equal(str_id_from_z("bin"), 0x39220) + assert_equal(str_id_from_z("archive"), 0x16C940E410) + + # Dash and the digits that fit the encoding + assert_equal(str_id_from_z("blk-incr"), 0x90DC9DAD820) + assert_equal(str_id_from_z("aes-256-cbc"), 0xC43DFBBCDCCA10) + + # The longest string the encoding holds + assert_equal(len("backup-error"), STRID5_MAX) + assert_equal(str_id_from_z("backup-error"), 0x93E522EE1558C220) + + +#################################################################################################################################### +def test_string_id_encode_6bit(): + """6-bit encoding is used when a string does not fit 5-bit, i.e. it has upper case or a digit outside 2, 5, and 6.""" + + assert_equal(str_id_from_z("i32"), 0x1E7C91) + assert_equal(str_id_from_z("lz4"), 0x2068C1) + assert_equal(str_id_from_z("execId"), 0x12E0C56051) + + assert_equal(str_id_from_z("lz4-dcmp"), 0x40D0C46E068C1) + + +#################################################################################################################################### +def test_string_id_encode_sequence(): + """A sequence below six is held in the header and above that in the high bits.""" + + # Held in the header + assert_equal(str_id_from_z("asc", 1), 0xE614) + assert_equal(str_id_from_z("auto", 1), 0x7D2A14) + assert_equal(str_id_from_z("auto", 2), 0x7D2A16) + assert_equal(str_id_from_z("count", 0), 0x14755E32) + assert_equal(str_id_from_z("debug", 5), 0x7A88A4C) + + # Held in the high bits + assert_equal(str_id_from_z("time", 6), 0x56A680E) + assert_equal(str_id_from_z("lsn", 7), 0x74D81E) + assert_equal(str_id_from_z("xid", 8), 0x22702E) + + +#################################################################################################################################### +def test_string_id_sequence_read(): + """The sequence can be read back from an encoded value.""" + + for value, sequence in (("asc", 1), ("count", 0), ("debug", 5), ("time", 6), ("lsn", 7), ("xid", 8)): + assert_equal(str_id_seq(str_id_from_z(value, sequence)), sequence, value) + + +#################################################################################################################################### +def test_string_id_encode_error(): + """A string that neither encoding can hold is an error.""" + + # Characters outside both encodings + for value in ("under_score", "dot.dot", "spa ce", "plus+"): + with assert_raises(TestError) as error: + str_id_from_z(value) + + assert_equal(str(error.exception), "'%s' contains invalid characters" % value) + + # Longer than 5-bit holds, and 6-bit holds fewer characters still so it cannot take over + assert_not_equal(str_id_from_z("a" * STRID5_MAX), 0) + + with assert_raises(TestError): + str_id_from_z("a" * (STRID5_MAX + 1)) + + # Upper case forces 6-bit, which holds fewer characters + assert_not_equal(str_id_from_z("A" * STRID6_MAX), 0) + + with assert_raises(TestError): + str_id_from_z("A" * (STRID6_MAX + 1)) + + # An empty string has nothing to encode + with assert_raises(TestError) as error: + str_id_from_z("") + + assert_equal(str(error.exception), "StringId value may not be empty") diff --git a/test/src/module/test/config/cli_test.py b/test/src/module/test/config/cli_test.py new file mode 100644 index 000000000..766d78981 --- /dev/null +++ b/test/src/module/test/config/cli_test.py @@ -0,0 +1,170 @@ +"""Test Command Line Interface.""" + +#################################################################################################################################### +import io +from contextlib import redirect_stderr, redirect_stdout + +from harness.test import * + +from config.cli import * + +# Version used for the parse, which only ever appears in the version output +VERSION = "2.60.0" + + +#################################################################################################################################### +def _cli_parse_exit(arg_list): + """Parse a command line that does not parse, returning the exit status and what argparse wrote. + + argparse reports and exits rather than raising, so its output is captured to keep it out of the test output.""" + + output = io.StringIO() + + with assert_raises(SystemExit) as error: + with redirect_stdout(output), redirect_stderr(output): + cli_parse(arg_list, VERSION) + + return error.exception.code, output.getvalue() + + +#################################################################################################################################### +def test_cli_unit_default(): + """The unit command defaults to what a plain local run needs.""" + + config = cli_parse(["unit", "common/error"], VERSION) + + assert_equal(config.command, "unit") + assert_equal(config.module, "common/error") + + # Shared by every command + assert_equal(config.repo_path, "pgbackrest") + assert_equal(config.log_level, "info") + assert_true(config.log_timestamp) + + # Shared by the commands that work in the test path + assert_equal(config.test_path, "test") + assert_equal(config.vm, "none") + + # Specific to the unit command + assert_equal(config.vm_id, 0) + assert_is_none(config.vm_arch) + assert_equal(config.pg_version, "invalid") + assert_is_none(config.test) + assert_is_none(config.test_name) + assert_is_none(config.coverage_file) + assert_equal(config.scale, 1) + assert_is_none(config.tz) + assert_equal(config.log_level_test, "off") + assert_true(config.coverage) + assert_true(config.back_trace) + assert_false(config.optimize) + assert_false(config.profile) + + +#################################################################################################################################### +def test_cli_unit_option(): + """Every unit option is accepted, which is how the Perl test framework drives a build.""" + + config = cli_parse( + [ + "unit", + "common/error", + "--repo-path=/repo", + "--log-level=detail", + "--no-log-timestamp", + "--test-path=/test", + "--vm=u22", + "--vm-id=2", + "--vm-arch=aarch64", + "--pg-version=16", + "--test=3", + "--test-name=test_one,test_two", + "--coverage-file=/test/raw.json", + "--scale=4", + "--tz=UTC", + "--log-level-test=debug", + "--no-coverage", + "--no-back-trace", + "--optimize", + "--profile", + ], + VERSION, + ) + + assert_equal(config.repo_path, "/repo") + assert_equal(config.log_level, "detail") + assert_false(config.log_timestamp) + assert_equal(config.test_path, "/test") + assert_equal(config.vm, "u22") + assert_equal(config.vm_id, 2) + assert_equal(config.vm_arch, "aarch64") + assert_equal(config.pg_version, "16") + assert_equal(config.test, 3) + assert_equal(config.test_name, "test_one,test_two") + assert_equal(config.coverage_file, "/test/raw.json") + assert_equal(config.scale, 4) + assert_equal(config.tz, "UTC") + assert_equal(config.log_level_test, "debug") + assert_false(config.coverage) + assert_false(config.back_trace) + assert_true(config.optimize) + assert_true(config.profile) + + +#################################################################################################################################### +def test_cli_coverage(): + """The coverage command takes every module that was run.""" + + config = cli_parse(["coverage", "common/error", "common/log"], VERSION) + + assert_equal(config.command, "coverage") + assert_equal(config.module, ["common/error", "common/log"]) + assert_false(config.coverage_summary) + + # It works in the test path so it has those options as well + assert_equal(config.test_path, "test") + + config = cli_parse(["coverage", "common/error", "--coverage-summary"], VERSION) + + assert_true(config.coverage_summary) + + +#################################################################################################################################### +def test_cli_lint(): + """The lint command reads the repository only, so it has none of the test path options.""" + + config = cli_parse(["lint", "--repo-path=/repo"], VERSION) + + assert_equal(config.command, "lint") + assert_equal(config.repo_path, "/repo") + assert_false(hasattr(config, "test_path")) + assert_false(hasattr(config, "vm")) + + +#################################################################################################################################### +def test_cli_error(): + """A command line that does not parse reports and exits.""" + + # A command is required since there is no default one + status, output = _cli_parse_exit([]) + + assert_equal(status, 2) + assert_in("the following arguments are required: command", output) + + # A level outside the log levels is rejected here rather than later when it is converted + status, output = _cli_parse_exit(["unit", "common/error", "--log-level=bogus"]) + + assert_equal(status, 2) + assert_in("invalid choice: 'bogus'", output) + + # The unit command takes exactly one module + status, output = _cli_parse_exit(["unit"]) + + assert_equal(status, 2) + assert_in("the following arguments are required: module/test", output) + + # The version is the project version rather than one of its own + status, output = _cli_parse_exit(["--version"]) + + assert_equal(status, 0) + assert_equal(output, "pgBackRest %s Test Harness\n" % VERSION) diff --git a/test/src/module/test/config/config_test.py b/test/src/module/test/config/config_test.py new file mode 100644 index 000000000..a782c7501 --- /dev/null +++ b/test/src/module/test/config/config_test.py @@ -0,0 +1,115 @@ +"""Test Configuration Load.""" + +#################################################################################################################################### +import os +import tempfile + +from harness.test import * + +from common.error import * +from common.log import * +from config.config import * + +# The version header as the project writes it, i.e. the define padded out to the comment column +VERSION = "2.60.0dev" +VERSION_H = '#define PROJECT_VERSION "%s"\n' % VERSION + + +#################################################################################################################################### +def _version_write(path, content=VERSION_H): + """Write the version header, which is the only part of the repository the config load reads.""" + + os.mkdir(os.path.join(path, "src")) + + with open(os.path.join(path, "src/version.h"), "w") as file: + file.write(content) + + +#################################################################################################################################### +def _cfg_load(arg_list, path): + """Load a configuration and put the log settings back, since loading sets them for the whole harness.""" + + try: + return cfg_load(arg_list, path) + finally: + log_init(INFO, True) + + +#################################################################################################################################### +def test_config_version(): + """The version comes from the repository the harness was checked out into.""" + + with tempfile.TemporaryDirectory() as path: + _version_write(path) + + assert_equal(project_version(path), VERSION) + + +#################################################################################################################################### +def test_config_version_error(): + """A version header without the define is an error rather than a version of nothing.""" + + with tempfile.TemporaryDirectory() as path: + # The define must be at the start of a line so a mention of it in a comment is not read as the version + _version_write(path, '// #define PROJECT_VERSION "9.9"\n#define PROJECT_NAME "pgbackrest"\n') + + with assert_raises(TestError) as error: + project_version(path) + + assert_equal(str(error.exception), "unable to find PROJECT_VERSION in src/version.h") + + +#################################################################################################################################### +def test_config_load(): + """A command line is parsed and what cannot be expressed in the parser is applied.""" + + with tempfile.TemporaryDirectory() as path: + _version_write(path) + + config = _cfg_load( + ["unit", "common/error", "--repo-path=%s" % path, "--test-path=%s/test" % path, "--log-level=detail"], path + ) + + assert_equal(config.command, "unit") + assert_equal(config.repo_path, path) + assert_equal(config.test_path, os.path.join(path, "test")) + + # Levels are ids by the time the load is done, since the parser has already checked that the names are valid + assert_equal(config.log_level, DETAIL) + assert_equal(config.log_level_test, OFF) + + # Paths are normalized so the harness does not depend on the working directory + config = _cfg_load(["unit", "common/error", "--repo-path=%s/." % path, "--test-path=%s/test/.." % path], path) + + assert_equal(config.repo_path, path) + assert_equal(config.test_path, path) + + +#################################################################################################################################### +def test_config_load_lint(): + """The lint command has neither a test path nor a test log level, so neither is applied.""" + + with tempfile.TemporaryDirectory() as path: + _version_write(path) + + config = _cfg_load(["lint", "--repo-path=%s" % path], path) + + assert_equal(config.command, "lint") + assert_equal(config.log_level, INFO) + assert_false(hasattr(config, "test_path")) + assert_false(hasattr(config, "log_level_test")) + + +#################################################################################################################################### +def test_config_load_error(): + """A repository that is not there is reported here rather than looking like a run that found nothing.""" + + with tempfile.TemporaryDirectory() as path: + _version_write(path) + + path_missing = os.path.join(path, "missing") + + with assert_raises(TestError) as error: + _cfg_load(["lint", "--repo-path=%s" % path_missing], path) + + assert_equal(str(error.exception), "repo path '%s' does not exist" % path_missing) diff --git a/test/src/module/test/coverageTest.c b/test/src/module/test/coverageTest.c deleted file mode 100644 index 87457b0e7..000000000 --- a/test/src/module/test/coverageTest.c +++ /dev/null @@ -1,567 +0,0 @@ -/*********************************************************************************************************************************** -Test Coverage Testing and Reporting -***********************************************************************************************************************************/ -#include "storage/posix/storage.h" - -#include "harness/storage.h" - -/*********************************************************************************************************************************** -Test Run -***********************************************************************************************************************************/ -static void -testRun(void) -{ - FUNCTION_HARNESS_VOID(); - - const Storage *const storageTest = storagePosixNewP(STRDEF(TEST_PATH), .write = true); - - // ***************************************************************************************************************************** - if (testBegin("testCvgGenerate()")) - { - const String *const pathRepo = STRDEF(TEST_PATH "/repo"); - const String *const pathTest = STRDEF(TEST_PATH "/test"); - - HRN_STORAGE_PUT_Z( - storageTest, "repo/test/define.yaml", - "unit:\n" - " - name: common\n" - " test:\n" - " - name: none\n" - " total: 1\n" - "\n" - " - name: error\n" - " total: 1\n" - " coverage:\n" - " - common/error/sub/error.inc: included\n" - " - common/error/sub/error.vendor: included\n" - " - doc/common/error/error: included\n" - " - test/common/error/error\n" - " - test/noCode: noCode\n" - "\n" - " - name: log\n" - " total: 1\n" - " coverage:\n" - " - common/log\n" - " - common/log2\n" - " - test/common/error/error\n" - "\n" - "integration:\n" - " - name: integration\n" - " test:\n" - " - name: all\n" - " total: 2\n" - "\n" - "performance:\n" - " - name: performance\n" - " test:\n" - " - name: type\n" - " total: 1\n"); - - HRN_STORAGE_PUT_Z( - storageTest, "test/repo/test/src/common/error/error.c", - /* 01 */ "FN_EXTERN const ErrorType *\n" - /* 02 */ "errorCodeIsError(const int code)\n" - /* 03 */ "{\n" - /* 04 */ " if (code == 0)\n" - /* 05 */ " return false;\n" - /* 06 */ "\n" - /* 07 */ " return true;\n" - /* 08 */ "}\n"); - - HRN_STORAGE_PUT_Z( - storageTest, "test/repo/doc/src/common/error/error.c", - /* 01 */ "int\n" - /* 02 */ "returnCode(int code)\n" - /* 03 */ "{\n" - /* 04 */ " return code;\n" - /* 05 */ "}\n"); - - HRN_STORAGE_PUT_Z( - storageTest, "test/repo/src/common/error/sub/error.c.inc", - /* 01 */ "int\n" - /* 02 */ "returnCodeInc(int code)\n" - /* 03 */ "{\n" - /* 04 */ " return code;\n" - /* 05 */ "}\n"); - - HRN_STORAGE_PUT_Z( - storageTest, "test/repo/src/common/error/sub/error.vendor.c.inc", - /* 01 */ "int\n" - /* 02 */ "returnCode(int code)\n" - /* 03 */ "{\n" - /* 04 */ " return code;\n" - /* 05 */ "}\n"); - - HRN_STORAGE_PUT_Z( - storageTest, "repo/test/result/coverage/raw/common-type-error.json", - "{" - // {uncrustify_off - indentation} - "\"gcc_version\": \"11.4.0\"," - "\"files\": [" - "{" - "\"lines\": [" - "{" - "\"branches\": [" - "{" - "\"count\": 1" - "}," - "{" - "\"count\": 0" - "}" - "]," - "\"count\": 1," - "\"line_number\": 4" - "}," - "{" - "\"count\": 1," - "\"line_number\": 5" - "}," - "{" - "\"count\": 0," - "\"line_number\": 7" - "}" - "]," - "\"functions\": [" - "{" - "\"start_line\": 2," - "\"name\": \"errorCodeIsError\"," - "\"execution_count\": 1," - "\"end_line\": 8" - "}" - "]," - "\"file\": \"../../../repo/test/src/common/error/error.c\"" - "}," - "{" - "\"lines\": [" - "{" - "\"count\": 1," - "\"line_number\": 4" - "}" - "]," - "\"functions\": [" - "{" - "\"start_line\": 2," - "\"name\": \"returnCode\"," - "\"execution_count\": 1," - "\"end_line\": 5" - "}" - "]," - "\"file\": \"../../../repo/test/src/common/error/sub/error.c.inc\"" - "}," - "{" - "\"lines\": [" - "{" - "\"count\": 1," - "\"line_number\": 4" - "}" - "]," - "\"functions\": [" - "{" - "\"start_line\": 2," - "\"name\": \"returnCodeInc\"," - "\"execution_count\": 1," - "\"end_line\": 5" - "}" - "]," - "\"file\": \"../../../repo/test/src/common/error/sub/error.vendor.c.inc\"" - "}," - "{" - "\"lines\": [" - "{" - "\"count\": 1," - "\"line_number\": 4" - "}" - "]," - "\"functions\": [" - "{" - "\"start_line\": 2," - "\"name\": \"returnCode\"," - "\"execution_count\": 1," - "\"end_line\": 5" - "}" - "]," - "\"file\": \"../../../repo/doc/src/common/error/error.c\"" - "}" - "]" - // {uncrustify_on} - "}"); - - HRN_STORAGE_PUT_Z( - storageTest, "test/repo/src/common/log.c", - /* 01 */ "static void\n" - /* 02 */ "logAnySet(void)\n" - /* 03 */ "{\n" - /* 04 */ " FUNCTION_TEST_VOID();\n" - /* 05 */ "\n" - /* 06 */ " logLevelAny = logLevelStdOut;\n" - /* 07 */ "\n" - /* 08 */ " if (logLevelStdErr > logLevelAny)\n" - /* 09 */ " FUNCTION_LOG_RETURN_VOID();\n" - /* 10 */ "\n" - /* 11 */ " if (logLevelFile > logLevelAny && logFdFile != -1)\n" - /* 12 */ " logLevelAny = logLevelFile;\n" - /* 13 */ "\n" - /* 14 */ " FUNCTION_TEST_RETURN_VOID();\n" - /* 15 */ "}\n" - /* 16 */ "\n" - /* 17 */ "FN_EXTERN bool\n" - /* 18 */ "logAny(const LogLevel logLevel)\n" - /* 19 */ "{\n" - /* 20 */ " FUNCTION_TEST_BEGIN();\n" - /* 21 */ " FUNCTION_TEST_PARAM(ENUM, logLevel);\n" - /* 22 */ " FUNCTION_TEST_END();\n" - /* 23 */ "\n" - /* 24 */ " ASSERT(logLevel);\n" - /* 25 */ "\n" - /* 26 */ " FUNCTION_TEST_RETURN(BOOL, logLevel); // {uncovered - x}\n" - /* 27 */ " x = 1;\n" - /* 28 */ "\n" - /* 29 */ "{\n" - /* 30 */ "}\n" - /* 31 */ "\n" - /* 32 */ "\n" - /* 33 */ "\n" - /* 34 */ "\n" - /* 35 */ "{\n" - /* 36 */ "}\n" - /* 37 */ "\n" - /* 38 */ " x = 2;\n" - /* 39 */ "}\n"); - - HRN_STORAGE_PUT_Z( - storageTest, "test/repo/src/common/log2.c", - /* 01 */ "int\n" - /* 02 */ "returnCode(int code)\n" - /* 03 */ "{\n" - /* 04 */ " if (code == 0)\n" - /* 05 */ " return code;\n" - /* 06 */ "}\n"); - - HRN_STORAGE_PUT_Z( - storageTest, "repo/test/result/coverage/raw/common-log.json", - "{" - // {uncrustify_off - indentation} - "\"gcc_version\": \"11.4.0\"," - "\"files\": [" - "{" - "\"lines\": [" - "{" - "\"branches\": [" - "{" - "\"count\": 0" - "}," - "{" - "\"count\": 1" - "}" - "]," - "\"count\": 1," - "\"line_number\": 4" - "}," - "{" - "\"count\": 0," - "\"line_number\": 5" - "}," - "{" - "\"count\": 1," - "\"line_number\": 7" - "}" - "]," - "\"functions\": [" - "{" - "\"start_line\": 2," - "\"name\": \"errorCodeIsError\"," - "\"execution_count\": 1," - "\"end_line\": 8" - "}" - "]," - "\"file\": \"../../../repo/test/src/common/error/error.c\"" - "}," - "{" - "\"lines\": [" - "{" - "\"branches\": []," - "\"count\": 0," - "\"line_number\": 4" - "}," - "{" - "\"branches\": []," - "\"count\": 1," - "\"line_number\": 6" - "}," - "{" - "\"branches\": [" - "{" - "\"fallthrough\": true," - "\"count\": 1" - "}," - "{" - "\"count\": 0" - "}" - "]," - "\"count\": 7," - "\"junk\": 0," - "\"line_number\": 8" - "}," - "{" - "\"branches\": []," - "\"count\": 1," - "\"line_number\": 9" - "}," - "{" - "\"branches\": []," - "\"count\": 1," - "\"line_number\": 14" - "}," - "{" - "\"branches\": []," - "\"count\": 1," - "\"line_number\": 21" - "}," - "{" - "\"branches\": []," - "\"count\": 0," - "\"line_number\": 22" - "}," - "{" - "\"branches\": [" - "{" - "\"count\": 1" - "}," - "{" - "\"count\": 0" - "}" - "]," - "\"count\": 1," - "\"line_number\": 24" - "}," - "{" - "\"branches\": []," - "\"count\": 0," - "\"line_number\": 26" - "}," - "{" - "\"branches\": []," - "\"count\": 0," - "\"line_number\": 27" - "}," - "{" - "\"branches\": []," - "\"count\": 0," - "\"line_number\": 38" - "}" - "]," - "\"functions\": [" - "{" - "\"blocks\": 18," - "\"start_line\": 2," - "\"name\": \"logAnySet\"," - "\"execution_count\": 1," - "\"end_line\": 15" - "}," - "{" - "\"blocks\": 18," - "\"start_line\": 18," - "\"name\": \"logAny\"," - "\"execution_count\": 0," - "\"end_line\": 33" - "}" - "]," - "\"junk\": 0," - "\"file\": \"../../../repo/src/common/log.c\"" - "}," - "{" - "\"lines\": [" - "{" - "\"branches\": [" - "{" - "\"count\": 0" - "}," - "{" - "\"count\": 1" - "}" - "]," - "\"count\": 1," - "\"line_number\": 4" - "}," - "{" - "\"count\": 1," - "\"line_number\": 5" - "}" - "]," - "\"functions\": [" - "{" - "\"start_line\": 2," - "\"name\": \"returnCode\"," - "\"execution_count\": 1," - "\"end_line\": 6" - "}" - "]," - "\"file\": \"../../../repo/test/src/common/log2.c\"" - "}" - "]" - // {uncrustify_on} - "}"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("common/none"); - - StringList *moduleList = strLstNew(); - strLstAddZ(moduleList, "common/none"); - - TEST_RESULT_INT(testCvgGenerate(pathRepo, pathTest, STRDEF("none"), false, moduleList), 0, "generate"); - - TEST_STORAGE_GET( - storageTest, "repo/test/result/coverage/coverage.html", - TEST_CVG_HTML_PRE - TEST_CVG_HTML_TOC_PRE - TEST_CVG_HTML_TOC_POST - TEST_CVG_HTML_POST); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("common/error"); - - moduleList = strLstNew(); - strLstAddZ(moduleList, "common/error"); - - TEST_RESULT_INT(testCvgGenerate(pathRepo, pathTest, STRDEF("vm"), false, moduleList), 0, "generate"); - TEST_RESULT_LOG("P00 WARN: module 'test/common/error/error' did not have all tests run required for coverage"); - - TEST_STORAGE_GET( - storageTest, "repo/test/result/coverage/coverage.html", - TEST_CVG_HTML_PRE - TEST_CVG_HTML_TOC_PRE - TEST_CVG_HTML_TOC_COVERED_PRE "doc/src/common/error/error.c" TEST_CVG_HTML_TOC_COVERED_POST - TEST_CVG_HTML_TOC_COVERED_PRE "src/common/error/sub/error.c.inc" TEST_CVG_HTML_TOC_COVERED_POST - TEST_CVG_HTML_TOC_COVERED_PRE "src/common/error/sub/error.vendor.c.inc" TEST_CVG_HTML_TOC_COVERED_POST - TEST_CVG_HTML_TOC_POST - TEST_CVG_HTML_POST); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("common/error and common/log summary"); - - moduleList = strLstNew(); - strLstAddZ(moduleList, "common/error"); - strLstAddZ(moduleList, "common/log"); - - TEST_RESULT_INT(testCvgGenerate(pathRepo, pathTest, STRDEF("vm"), true, moduleList), 2, "generate"); - TEST_RESULT_LOG( - "P00 WARN: module 'src/common/log.c' is not fully covered (4/9 lines, 1/2 branches)\n" - "P00 WARN: module 'src/common/log2.c' is not fully covered (2/2 lines, 1/2 branches)"); - - TEST_STORAGE_GET( - storageTest, "repo/doc/xml/auto/metric-coverage-report.auto.xml", - "\n" - " common\n" - " 2/3 (66.67%)\n" - " 2/4 (50.00%)\n" - " 6/11 (54.55%)\n" - "\n" - "\n" - "\n" - " common/error/sub\n" - " 2/2 (100.00%)\n" - " ---\n" - " 2/2 (100.00%)\n" - "\n" - "\n" - "\n" - " TOTAL\n" - " 4/5 (80.00%)\n" - " 2/4 (50.00%)\n" - " 8/13 (61.54%)\n" - "\n"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("common/error and common/log"); - - TEST_RESULT_INT(testCvgGenerate(pathRepo, pathTest, STRDEF("vm"), false, moduleList), 2, "generate"); - TEST_RESULT_LOG( - "P00 WARN: module 'src/common/log.c' is not fully covered (5/9 lines, 1/2 branches)\n" - "P00 WARN: module 'src/common/log2.c' is not fully covered (2/2 lines, 1/2 branches)"); - - TEST_STORAGE_GET( - storageTest, "repo/test/result/coverage/coverage.html", - TEST_CVG_HTML_PRE - - TEST_CVG_HTML_TOC_PRE - TEST_CVG_HTML_TOC_COVERED_PRE "doc/src/common/error/error.c" TEST_CVG_HTML_TOC_COVERED_POST - TEST_CVG_HTML_TOC_COVERED_PRE "src/common/error/sub/error.c.inc" TEST_CVG_HTML_TOC_COVERED_POST - TEST_CVG_HTML_TOC_COVERED_PRE "src/common/error/sub/error.vendor.c.inc" TEST_CVG_HTML_TOC_COVERED_POST - TEST_CVG_HTML_TOC_UNCOVERED_PRE "src/common/log.c" TEST_CVG_HTML_TOC_UNCOVERED_MID "src/common/log.c" - TEST_CVG_HTML_TOC_UNCOVERED_POST - TEST_CVG_HTML_TOC_UNCOVERED_PRE "src/common/log2.c" TEST_CVG_HTML_TOC_UNCOVERED_MID "src/common/log2.c" - TEST_CVG_HTML_TOC_UNCOVERED_POST - TEST_CVG_HTML_TOC_COVERED_PRE "test/src/common/error/error.c" TEST_CVG_HTML_TOC_COVERED_POST - TEST_CVG_HTML_TOC_POST - - TEST_CVG_HTML_RPT_PRE "src/common/log.c" TEST_CVG_HTML_RPT_MID1 "src/common/log.c" TEST_CVG_HTML_RPT_MID2 - TEST_CVG_HTML_RPT_LINE_PRE "1" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE "static void" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "2" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE "logAnySet(void)" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "3" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE "{" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "4" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE_UNCOVERED " FUNCTION_TEST_VOID();" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "5" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE "" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "6" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE " logLevelAny = logLevelStdOut;" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "7" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE "" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "8" TEST_CVG_HTML_RPT_BRANCH_UNCOVERED_PRE "[+ -]" TEST_CVG_HTML_RPT_BRANCH_UNCOVERED_POST - TEST_CVG_HTML_RPT_CODE " if (logLevelStdErr > logLevelAny)" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "9" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE " FUNCTION_LOG_RETURN_VOID();" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "10" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE "" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "11" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE " if (logLevelFile > logLevelAny && logFdFile != -1)" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "12" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE " logLevelAny = logLevelFile;" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_SKIP - TEST_CVG_HTML_RPT_LINE_PRE "17" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE "FN_EXTERN bool" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "18" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE "logAny(const LogLevel logLevel)" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "19" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE "{" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "20" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE " FUNCTION_TEST_BEGIN();" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "21" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE " FUNCTION_TEST_PARAM(ENUM, logLevel);" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "22" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE_UNCOVERED " FUNCTION_TEST_END();" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "23" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE "" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "24" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE " ASSERT(logLevel);" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "25" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE "" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "26" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE " FUNCTION_TEST_RETURN(BOOL, logLevel); // {uncovered - x}" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "27" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE_UNCOVERED " x = 1;" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_SKIP - TEST_CVG_HTML_RPT_LINE_PRE "38" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE_UNCOVERED " x = 2;" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_POST - - TEST_CVG_HTML_RPT_PRE "src/common/log2.c" TEST_CVG_HTML_RPT_MID1 "src/common/log2.c" TEST_CVG_HTML_RPT_MID2 - TEST_CVG_HTML_RPT_LINE_PRE "1" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE "int" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "2" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE "returnCode(int code)" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "3" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE "{" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "4" TEST_CVG_HTML_RPT_BRANCH_UNCOVERED_PRE "[- +]" TEST_CVG_HTML_RPT_BRANCH_UNCOVERED_POST - TEST_CVG_HTML_RPT_CODE " if (code == 0)" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_LINE_PRE "5" TEST_CVG_HTML_RPT_BRANCH_COVERED - TEST_CVG_HTML_RPT_CODE " return code;" TEST_CVG_HTML_RPT_LINE_POST - TEST_CVG_HTML_RPT_POST - - TEST_CVG_HTML_POST); - } - - FUNCTION_HARNESS_RETURN_VOID(); -} diff --git a/test/src/module/test/testTest.c b/test/src/module/test/testTest.c deleted file mode 100644 index 0ba9bec4a..000000000 --- a/test/src/module/test/testTest.c +++ /dev/null @@ -1,1278 +0,0 @@ -/*********************************************************************************************************************************** -Test Test Command -***********************************************************************************************************************************/ -#include "storage/posix/storage.h" - -#include "harness/storage.h" - -/*********************************************************************************************************************************** -Build list of test files to compare -***********************************************************************************************************************************/ -static StringList * -testStorageList(const Storage *const storage) -{ - StringList *const result = strLstNew(); - - MEM_CONTEXT_TEMP_BEGIN() - { - StorageIterator *storageItr = storageNewItrP(storage, NULL, .sortOrder = sortOrderAsc, .recurse = true); - - while (storageItrMore(storageItr)) - { - const StorageInfo info = storageItrNext(storageItr); - - if (info.type == storageTypeFile && !strBeginsWithZ(info.name, "build/")) - strLstAdd(result, info.name); - } - } - MEM_CONTEXT_TEMP_END(); - - return result; -} - -/*********************************************************************************************************************************** -Copy a list of files from the repo to the test repo -***********************************************************************************************************************************/ -#define TEST_COPY(source, destination, ...) \ - do \ - { \ - const char *const copyList[] = {__VA_ARGS__}; \ - \ - for (unsigned int copyIdx = 0; copyIdx < LENGTH_OF(copyList); copyIdx++) \ - { \ - HRN_STORAGE_PUT( \ - destination, zNewFmt("repo/%s", copyList[copyIdx]), storageGetP(storageNewReadP(source, STR(copyList[copyIdx])))); \ - } \ - } \ - while (0) - -/*********************************************************************************************************************************** -Test Run -***********************************************************************************************************************************/ -static void -testRun(void) -{ - FUNCTION_HARNESS_VOID(); - - const Storage *const storageRepo = storagePosixNewP(STRDEF(HRN_PATH_REPO)); - const Storage *const storageTest = storagePosixNewP(STRDEF(TEST_PATH), .write = true); - - // ***************************************************************************************************************************** - if (testBegin("cmdBldPathRelative() and cmdTestVmArchFix()")) - { - TEST_TITLE("cmdBldPathRelative()"); - - TEST_ERROR(cmdBldPathRelative(STRDEF("/tmp"), STRDEF("/tmp")), AssertError, "assertion '!strEq(base, compare)' failed"); - - TEST_RESULT_STR_Z(cmdBldPathRelative(STRDEF("/tmp/sub"), STRDEF("/tmp")), "..", "compare is sub of base"); - TEST_RESULT_STR_Z(cmdBldPathRelative(STRDEF("/tmp"), STRDEF("/tmp/sub")), "sub", "base is sub of compare"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("cmdTestVmArchFix()"); - - TEST_RESULT_STR_Z(cmdTestVmArchFix(STRDEF("i686")), "i386", "i686 -> i386"); - TEST_RESULT_STR_Z(cmdTestVmArchFix(STRDEF("arm64")), "aarch64", "arm64 -> aarch64"); - TEST_RESULT_STR_Z(cmdTestVmArchFix(STRDEF("x86_64")), "x86_64", "x86_64 -> x86_64"); - } - - // ***************************************************************************************************************************** - if (testBegin("lint*()")) - { - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("lintStrId()"); - - HRN_STORAGE_PUT_Z( - storageTest, "repo/test.c", - "#define STRID5(str, strId)\n" - "#define STRID5S(str, seq, strId)\n" - "#define STRID6(str, strId)\n" - "#define STRID6S(str, seq, strId)\n" - "STR" "ID5(\"abcd\", TEST_STRID)\n" - "STR" "ID5(\\\"abcd\\\")\n" - "STR" "ID5(\\\"abcd)\n" - "STR" "ID5(abcd)\n" - "STR" "ID5( \"abcd)\n" - "STRID5(\"abcd\")\n" - "STRID5(\"abcd\", 0x20c410)\n" - "STRID6(\"abcd\", 0x20c410)\n" - "STRID6S(\"abcd\", 37, 0x20c410)\n"); - - TEST_ERROR( - cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("none"), 3, STRDEF("invalid"), - STRDEF("common/stack-trace"), 0, 1, logLevelDebug, true, NULL, NULL, false, false, false, true), - FormatError, "6 linter error(s) in 'test.c' (see warnings above)"); - - TEST_RESULT_LOG( - "P00 WARN: 'STR" "ID5(\\\"abcd)' must have quotes around string parameter '\\\"abcd'\n" - "P00 WARN: 'STR" "ID5(abcd)' must have quotes around string parameter 'abcd'\n" - "P00 WARN: 'STR" "ID5( \"abcd)' must have quotes around string parameter '\"abcd'\n" - "P00 WARN: 'STRID5(\"abcd\")' should be 'STRID5(\"abcd\", 0x20c410)'\n" - "P00 WARN: 'STRID6(\"abcd\", 0x20c410)' should be 'STRID5(\"abcd\", 0x20c410)'\n" - "P00 WARN: 'STRID6S(\"abcd\", 37, 0x20c410)' should be 'STRID5S(\"abcd\", 37, 0x41883fe)'"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("lintChar()"); - - // Invisible/bidirectional/homoglyph/control characters written as \xNN escapes so this test file stays 7-bit ASCII - HRN_STORAGE_PUT_Z( - storageTest, "repo/test.c", - "int normal;\n" // Printable ASCII is allowed - "\tint tabbed;\n" // Tab is allowed - "int zero\xe2\x80\x8bwidth;\n" // U+200B zero width space - "int bidi\xe2\x80\xae;\n" // U+202E right-to-left override - "int cyr\xd0\xb0;\n" // U+0430 cyrillic homoglyph of 'a' - "int emoji\xf0\x9f\x98\x80;\n" // U+1F600 four-byte sequence - "int cr;\r\n" // U+000D carriage return - "int trunc\xf0"); // Truncated lead byte at end of file - - // A skip-listed file is exempt even though its content would otherwise be flagged - HRN_STORAGE_PUT_Z(storageTest, "repo/doc/resource/git-history.cache", "[]\n"); - - TEST_ERROR( - cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("none"), 3, STRDEF("invalid"), - STRDEF("common/stack-trace"), 0, 1, logLevelDebug, true, NULL, NULL, false, false, false, true), - FormatError, "6 linter error(s) in 'test.c' (see warnings above)"); - - TEST_RESULT_LOG( - "P00 WARN: line 3 contains disallowed character U+200B (source must be 7-bit ASCII)\n" - "P00 WARN: line 4 contains disallowed character U+202E (source must be 7-bit ASCII)\n" - "P00 WARN: line 5 contains disallowed character U+0430 (source must be 7-bit ASCII)\n" - "P00 WARN: line 6 contains disallowed character U+1F600 (source must be 7-bit ASCII)\n" - "P00 WARN: line 7 contains disallowed character U+000D (source must be 7-bit ASCII)\n" - "P00 WARN: line 8 contains disallowed character U+00F0 (source must be 7-bit ASCII)"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("lintChar() binary file"); - - // A binary file (contains a null byte) is not allowed; the skip-listed cache above is still exempt - HRN_STORAGE_PUT(storageTest, "repo/test.c", BUF("a\0b", 3)); - - TEST_ERROR( - cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("none"), 3, STRDEF("invalid"), - STRDEF("common/stack-trace"), 0, 1, logLevelDebug, true, NULL, NULL, false, false, false, true), - FormatError, "1 linter error(s) in 'test.c' (see warnings above)"); - - TEST_RESULT_LOG("P00 WARN: unexpected binary file (add to the linter skip list if intentional)"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("lintStrId() runs on .c.inc but not .auto.c.inc"); - - // Generated .auto.c.inc and vendored .vendor.c.inc are exempt from the StringId check, so their bad StringIds must not be - // flagged (both sort before b.c.inc) - HRN_STORAGE_PUT_Z(storageTest, "repo/a.auto.c.inc", "STRID5(\"abcd\")\n"); - HRN_STORAGE_PUT_Z(storageTest, "repo/a.vendor.c.inc", "STRID5(\"abcd\")\n"); - - // A hand-written .c.inc is checked, so its bad StringId is flagged - HRN_STORAGE_PUT_Z(storageTest, "repo/b.c.inc", "STRID5(\"abcd\")\n"); - - TEST_ERROR( - cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("none"), 3, STRDEF("invalid"), - STRDEF("common/stack-trace"), 0, 1, logLevelDebug, true, NULL, NULL, false, false, false, true), - FormatError, "1 linter error(s) in 'b.c.inc' (see warnings above)"); - - TEST_RESULT_LOG("P00 WARN: 'STRID5(\"abcd\")' should be 'STRID5(\"abcd\", 0x20c410)'"); - } - - // ***************************************************************************************************************************** - if (testBegin("TestDef and TestBuild")) - { - // meson_options.txt - // ------------------------------------------------------------------------------------------------------------------------- - const char *const mesonOption = strZ(strNewBuf(storageGetP(storageNewReadP(storageRepo, STRDEF("meson_options.txt"))))); - HRN_STORAGE_PUT_Z(storageTest, "repo/meson_options.txt", mesonOption); - - // Root meson.build - // ------------------------------------------------------------------------------------------------------------------------- - String *const mesonBuildRoot = strCat( - strNew(), strNewBuf(storageGetP(storageNewReadP(storageRepo, STRDEF("meson.build"))))); - - HRN_STORAGE_PUT_Z(storageTest, "repo/meson.build", strZ(mesonBuildRoot)); - strReplace(mesonBuildRoot, STRDEF("subdir('"), STRDEF("# subdir('")); - strReplace(mesonBuildRoot, STRDEF("meson.add_dist_script("), STRDEF("# meson.add_dist_script(")); - - strCatZ( - mesonBuildRoot, - "\n" - MESON_COMMENT_BLOCK "\n" - "# Write configuration\n" - MESON_COMMENT_BLOCK "\n" - "configure_file(output: 'build.auto.h', configuration: configuration)\n" - "\n" - "add_global_arguments('-DFN_EXTERN=extern', language : 'c')\n" - "add_global_arguments('-DVR_EXTERN_DECLARE=extern', language : 'c')\n" - "add_global_arguments('-DVR_EXTERN_DEFINE=', language : 'c')\n" - "add_global_arguments('-DERROR_MESSAGE_BUFFER_SIZE=131072', language : 'c')\n"); - - // harnessError.c - // ------------------------------------------------------------------------------------------------------------------------- - String *const harnessErrorC = strCat( - strNew(), strNewBuf(storageGetP(storageNewReadP(storageRepo, STRDEF("test/src/harness/error.c"))))); - - strReplace(harnessErrorC, STRDEF("{[SHIM_MODULE]}"), STRDEF("#include \"" TEST_PATH "/repo/src/common/error/error.c\"")); - - // Unit test harness - // ------------------------------------------------------------------------------------------------------------------------- - String *const testC = strCat(strNew(), strNewBuf(storageGetP(storageNewReadP(storageRepo, STRDEF("test/src/test.c"))))); - - strReplace(testC, STRDEF("{[C_HRN_PATH]}"), STRDEF(TEST_PATH "/test/data-3")); - strReplace(testC, STRDEF("{[C_HRN_PATH_REPO]}"), STRDEF(TEST_PATH "/repo")); - strReplace(testC, STRDEF("{[C_LOG_LEVEL_TEST]}"), STRDEF("logLevelDebug")); - strReplace(testC, STRDEF("{[C_TEST_GROUP]}"), STRDEF(TEST_GROUP)); - strReplace(testC, STRDEF("{[C_TEST_GROUP_ID]}"), STRDEF(TEST_GROUP_ID_Z)); - strReplace(testC, STRDEF("{[C_TEST_GROUP_ID_Z]}"), STRDEF("\"" TEST_GROUP_ID_Z "\"")); - strReplace(testC, STRDEF("{[C_TEST_IDX]}"), STRDEF("3")); - strReplace(testC, STRDEF("{[C_TEST_PATH]}"), STRDEF(TEST_PATH "/test/test-3")); - strReplace(testC, STRDEF("{[C_TEST_PGB_PATH]}"), STRDEF("../../../../repo")); - strReplace(testC, STRDEF("{[C_TEST_SCALE]}"), STRDEF("1")); - strReplace(testC, STRDEF("{[C_TEST_TIMING]}"), STRDEF("true")); - strReplace(testC, STRDEF("{[C_TEST_USER]}"), STRDEF(TEST_USER)); - strReplace(testC, STRDEF("{[C_TEST_USER_ID]}"), STRDEF(TEST_USER_ID_Z)); - strReplace(testC, STRDEF("{[C_TEST_USER_ID_Z]}"), STRDEF("\"" TEST_USER_ID_Z "\"")); - strReplace(testC, STRDEF("{[C_TEST_USER_LEN]}"), strNewFmt("%zu", sizeof(TEST_USER) - 1)); - strReplace(testC, STRDEF("{[C_TEST_ARCHITECTURE]}"), STRDEF(TEST_ARCHITECTURE)); - - const String *const architecture = - strEqZ(strTrim(execOneExpectP(STRDEF("uname -m"))), TEST_ARCHITECTURE) ? NULL : strNewZ(TEST_ARCHITECTURE); - - // Test definition - // ------------------------------------------------------------------------------------------------------------------------- - HRN_STORAGE_PUT_Z( - storageTest, "repo/test/define.yaml", - "unit:\n" - " - name: common\n" - " test:\n" - " - name: pre\n" - " total: 1\n" - "\n" - " - name: error\n" - " total: 9\n" - " feature: error\n" - " harness:\n" - " name: error\n" - " shim:\n" - " - common/error/error\n" - " coverage:\n" - " - common/error/error\n" - " - common/error/error.auto: noCode\n" - " - common/error/error.inc: included\n" - " depend:\n" - " - common/stackTrace\n" - " - common/type/stringStatic\n" - "\n" - " - name: stack-trace\n" - " total: 4\n" - " feature: stackTrace\n" - " harness:\n" - " name: stackTrace\n" - " integration: false\n" - " shim:\n" - " - name: common/stackTrace\n" - " function:\n" - " - stackTraceBackCallback\n" - " coverage:\n" - " - common/stackTrace\n" - " - common/type/stringStatic\n" - " depend:\n" - " - common/debug\n" - "\n" - " - name: test\n" - " test:\n" - " - name: shim\n" - " binReq: true\n" - " containerReq: true\n" - " total: 1\n" - " define: -DNDEBUG\n" - " harness:\n" - " - noShim\n" - " - name: shim\n" - " shim:\n" - " - name: test/common/shim\n" - " function:\n" - " - shimFunc:\n" - " inc: shimInc\n" - " - shimFunc2:\n" - " inc: shimInc\n" - " - test/common/shim2\n" - " coverage:\n" - " - test/common/shim\n" - " - test/common/shim2\n" - " - test/common/shimInc.inc: included\n" - " include:\n" - " - common/error/error\n" - " - test/common/include\n" - " - doc/command/build/build\n" - "\n" - "integration:\n" - " - name: integration\n" - " test:\n" - " - name: all\n" - " total: 2\n" - "\n" - " - name: real\n" - " db: true\n" - " test:\n" - " - name: all\n" - " total: 1\n" - "\n" - "performance:\n" - " - name: performance\n" - " test:\n" - " - name: type\n" - " total: 1\n"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("Test common/stack-trace"); - - TEST_COPY( - storageRepo, storageTest, - "src/build.h", - "src/common/assert.h", - "src/common/debug.c", - "src/common/debug.h", - "src/common/error/error.auto.c.inc", - "src/common/error/error.auto.h", - "src/common/error/error.c", - "src/common/error/error.h", - "src/common/logLevel.h", - "src/common/macro.h", - "src/common/stackTrace.c", - "src/common/stackTrace.h", - "src/common/type/convert.h", - "src/common/type/param.h", - "src/common/type/stringStatic.h", - "src/common/type/stringStatic.c", - "test/src/harness/debug.h", - "test/src/harness/log.h", - "test/src/harness/error.c", - "test/src/harness/error.h", - "test/src/harness/stackTrace.h", - "test/src/harness/stackTrace.c", - "test/src/harness/test.c", - "test/src/harness/test.h", - "test/src/harness/test.intern.h", - "test/src/module/common/stackTraceTest.c", - "test/src/test.c"); - - HRN_STORAGE_PUT_EMPTY(storageTest, "repo/doc/src/command/build/build.c"); - - TEST_RESULT_VOID( - cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("none"), 3, STRDEF("invalid"), - STRDEF("common/stack-trace"), 0, 1, logLevelDebug, true, NULL, architecture, false, false, false, true), - "new build"); - - // Older versions of ninja may error on a rebuild so a retry may occur - TEST_RESULT_LOG_EMPTY_OR_CONTAINS("WARN: build failed for unit"); - - const Storage *storageUnit = storagePosixNewP(STRDEF(TEST_PATH "/test/unit-3/none")); - StringList *fileList = testStorageList(storageUnit); - - TEST_RESULT_STRLST_Z( - fileList, - "meson.build\n" - "meson_options.txt\n" - "src/common/stackTrace.c\n" - "test/src/harness/error.c\n" - "test/src/harness/stackTrace.c\n" - "test.c\n", - "check files"); - - for (unsigned int fileIdx = 0; fileIdx < strLstSize(fileList); fileIdx++) - { - const String *const file = strLstGet(fileList, fileIdx); - - if (strEqZ(file, "meson.build")) - { - TEST_STORAGE_GET( - storageUnit, strZ(file), - zNewFmt( - "%s" - "add_global_arguments('-DHRN_INTEST_STACKTRACE', language : 'c')\n" - "add_global_arguments('-DHRN_FEATURE_ERROR', language : 'c')\n" - "\n" - MESON_COMMENT_BLOCK "\n" - "# Unit test\n" - MESON_COMMENT_BLOCK "\n" - "src_unit = files(\n" - " '../../../repo/src/common/debug.c',\n" - " 'test/src/harness/error.c',\n" - " '../../../repo/test/src/harness/test.c',\n" - " 'test.c',\n" - ")\n" - "\n" - "executable(\n" - " 'test-unit',\n" - " sources: src_unit,\n" - " include_directories:\n" - " include_directories(\n" - " '.',\n" - " '../../../repo/src',\n" - " '../../../repo/doc/src',\n" - " '../../../repo/test/src',\n" - " ),\n" - " dependencies: [\n" - " lib_backtrace,\n" - " lib_bz2,\n" - " lib_openssl,\n" - " lib_lz4,\n" - " lib_pq,\n" - " lib_ssh2,\n" - " lib_xml,\n" - " lib_yaml,\n" - " lib_z,\n" - " lib_zstd,\n" - " ],\n" - ")\n", - strZ(mesonBuildRoot))); - } - else if (strEqZ(file, "meson_options.txt")) - { - TEST_STORAGE_GET(storageUnit, strZ(file), mesonOption); - } - else if (strEqZ(file, "src/common/stackTrace.c") || strEqZ(file, "test/src/harness/stackTrace.c")) - { - // No test needed - } - else if (strEqZ(file, "test/src/harness/error.c")) - { - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(harnessErrorC)); - } - else if (strEqZ(file, "test.c")) - { - String *const testCDup = strCat(strNew(), testC); - - strReplace(testCDup, STRDEF("{[C_TEST_CONTAINER]}"), STRDEF("false")); - strReplace(testCDup, STRDEF("{[C_TEST_VM]}"), STRDEF("none")); - strReplace(testCDup, STRDEF("{[C_TEST_PG_VERSION]}"), STRDEF("invalid")); - strReplace(testCDup, STRDEF("{[C_TEST_LOG_EXPECT]}"), STRDEF("true")); - strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("#define DEBUG_TEST_TRACE")); - strReplace(testCDup, STRDEF("{[C_TEST_PATH_BUILD]}"), STRDEF(TEST_PATH "/test/unit-3/none/build")); - strReplace(testCDup, STRDEF("{[C_TEST_PROFILE]}"), STRDEF("false")); - strReplace(testCDup, STRDEF("{[C_TEST_PROJECT_EXE]}"), STRDEF(TEST_PATH "/test/build/none/src/pgbackrest")); - strReplace(testCDup, STRDEF("{[C_TEST_TZ]}"), STRDEF("// No timezone specified")); - - strReplace( - testCDup, STRDEF("{[C_INCLUDE]}"), - STRDEF( - "#include \"test/src/harness/stackTrace.c\"\n" - "#include \"../../../repo/src/common/type/stringStatic.c\"")); - strReplace(testCDup, STRDEF("{[C_INCLUDE]}"), STRDEF("#include \"test/src/harness/stackTrace.c\"")); - strReplace( - testCDup, STRDEF("{[C_TEST_INCLUDE]}"), - STRDEF("#include \"../../../repo/test/src/module/common/stackTraceTest.c\"")); - strReplace( - testCDup, STRDEF("{[C_TEST_LIST]}"), - STRDEF( - "hrnAdd( 1, true);\n" - " hrnAdd( 2, true);\n" - " hrnAdd( 3, true);\n" - " hrnAdd( 4, true);")); - - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(testCDup)); - } - else - THROW_FMT(TestError, "no test for '%s'", strZ(file)); - } - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("Test common/error"); - - TEST_COPY( - storageRepo, storageTest, - "test/src/harness/fork.h", - "test/src/module/common/errorTest.c"); - - TEST_RESULT_VOID( - cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("none"), 3, STRDEF("invalid"), - STRDEF("common/error"), 5, 1, logLevelDebug, true, NULL, architecture, false, false, false, true), - "new build"); - - // Older versions of ninja may error on a rebuild so a retry may occur - TEST_RESULT_LOG_EMPTY_OR_CONTAINS("WARN: build failed for unit"); - - fileList = testStorageList(storageUnit); - - TEST_RESULT_STRLST_Z( - fileList, - "meson.build\n" - "meson_options.txt\n" - "test/src/harness/error.c\n" - "test.c\n", - "check files"); - - for (unsigned int fileIdx = 0; fileIdx < strLstSize(fileList); fileIdx++) - { - const String *const file = strLstGet(fileList, fileIdx); - - if (strEqZ(file, "meson.build")) - { - TEST_STORAGE_GET( - storageUnit, strZ(file), - zNewFmt( - "%s" - "add_global_arguments('-DHRN_INTEST_ERROR', language : 'c')\n" - "\n" - MESON_COMMENT_BLOCK "\n" - "# Unit test\n" - MESON_COMMENT_BLOCK "\n" - "src_unit = files(\n" - " '../../../repo/src/common/stackTrace.c',\n" - " '../../../repo/src/common/type/stringStatic.c',\n" - " '../../../repo/test/src/harness/test.c',\n" - " 'test.c',\n" - ")\n" - "\n" - "executable(\n" - " 'test-unit',\n" - " sources: src_unit,\n" - " include_directories:\n" - " include_directories(\n" - " '.',\n" - " '../../../repo/src',\n" - " '../../../repo/doc/src',\n" - " '../../../repo/test/src',\n" - " ),\n" - " dependencies: [\n" - " lib_backtrace,\n" - " lib_bz2,\n" - " lib_openssl,\n" - " lib_lz4,\n" - " lib_pq,\n" - " lib_ssh2,\n" - " lib_xml,\n" - " lib_yaml,\n" - " lib_z,\n" - " lib_zstd,\n" - " ],\n" - ")\n", - strZ(mesonBuildRoot))); - } - else if (strEqZ(file, "meson_options.txt")) - { - TEST_STORAGE_GET(storageUnit, strZ(file), mesonOption); - } - else if (strEqZ(file, "test/src/harness/error.c")) - { - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(harnessErrorC)); - } - else if (strEqZ(file, "test.c")) - { - String *const testCDup = strCat(strNew(), testC); - - strReplace(testCDup, STRDEF("{[C_TEST_CONTAINER]}"), STRDEF("false")); - strReplace(testCDup, STRDEF("{[C_TEST_VM]}"), STRDEF("none")); - strReplace(testCDup, STRDEF("{[C_TEST_PG_VERSION]}"), STRDEF("invalid")); - strReplace(testCDup, STRDEF("{[C_TEST_LOG_EXPECT]}"), STRDEF("true")); - strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("#define DEBUG_TEST_TRACE")); - strReplace(testCDup, STRDEF("{[C_TEST_PATH_BUILD]}"), STRDEF(TEST_PATH "/test/unit-3/none/build")); - strReplace(testCDup, STRDEF("{[C_TEST_PROFILE]}"), STRDEF("false")); - strReplace(testCDup, STRDEF("{[C_TEST_PROJECT_EXE]}"), STRDEF(TEST_PATH "/test/build/none/src/pgbackrest")); - strReplace(testCDup, STRDEF("{[C_TEST_TZ]}"), STRDEF("// No timezone specified")); - - strReplace( - testCDup, STRDEF("{[C_INCLUDE]}"), - STRDEF("#include \"test/src/harness/error.c\"")); - strReplace( - testCDup, STRDEF("{[C_TEST_INCLUDE]}"), - STRDEF("#include \"../../../repo/test/src/module/common/errorTest.c\"")); - strReplace( - testCDup, STRDEF("{[C_TEST_LIST]}"), - STRDEF( - "hrnAdd( 1, false);\n" - " hrnAdd( 2, false);\n" - " hrnAdd( 3, false);\n" - " hrnAdd( 4, false);\n" - " hrnAdd( 5, true);\n" - " hrnAdd( 6, false);\n" - " hrnAdd( 7, false);\n" - " hrnAdd( 8, false);\n" - " hrnAdd( 9, false);")); - - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(testCDup)); - } - else - THROW_FMT(TestError, "no test for '%s'", strZ(file)); - } - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("Test test/shim"); - - String *const shimC = strCatZ( - strNew(), - "#include \"test/common/shimInc.c.inc\"\n"); - - HRN_STORAGE_PUT_Z(storageTest, "repo/test/src/common/shim.c", strZ(shimC)); - - // shimFunc/shimFunc2 are defined in an included module to test shimming functions that are not in the shim module itself - String *const shimIncC = strCatZ( - strNew(), - "int\n" - "shimFunc(void)\n" - "{\n" - " return 777;\n" - "}\n" - "\n" - "static int\n" - "shimFunc2(\n" - " int param1,\n" - " int param2)\n" - "{\n" - " return 777 + param1 + param2;\n" - "}\n"); - - HRN_STORAGE_PUT_Z(storageTest, "repo/test/src/common/shimInc.c.inc", strZ(shimIncC)); - - HRN_STORAGE_PUT_Z( - storageTest, "repo/test/src/common/shim2.c", - "int noShimFunc3(void);" - "int noShimFunc3(void)\n" - "{\n" - " return 888;\n" - "}\n"); - - strReplace(shimIncC, STRDEF("int\nshimFunc(void)"), STRDEF("int shimFunc_SHIMMED(void); int\nshimFunc_SHIMMED(void)")); - strReplace( - shimIncC, STRDEF("static int\nshimFunc2("), - STRDEF("static int shimFunc2(int param1, int param2); static int\nshimFunc2_SHIMMED(")); - - String *const harnessShimC = strCatZ( - strNew(), - "{[SHIM_MODULE]}\n" - "\n" - "static int\n" - "shimFunc(void)\n" - "{\n" - " return shimFunc_SHIMMED() + 1;\n" - " (void)shimFunc2; // To suppress unused warnings\n" - "}\n" - "\n" - "static int\n" - "shimFunc2(int param1, int param2)\n" - "{\n" - " return shimFunc2_SHIMMED(param1, param2) + 2;\n" - " (void)shimFunc; // To suppress unused warnings\n" - "}\n"); - - HRN_STORAGE_PUT_Z(storageTest, "repo/test/src/harness/shim.c", strZ(harnessShimC)); - HRN_STORAGE_PUT_EMPTY(storageTest, "repo/test/src/harness/shim/sub.c"); - HRN_STORAGE_PUT_EMPTY(storageTest, "repo/test/src/harness/noShim.c"); - HRN_STORAGE_PUT_EMPTY(storageTest, "repo/test/src/common/include.c"); - - strReplace( - harnessShimC, STRDEF("{[SHIM_MODULE]}"), - STRDEF( - "#include \"" TEST_PATH "/test/unit-3/uXX/test/src/common/shim.c\"\n" - "#include \"" TEST_PATH "/repo/test/src/common/shim2.c\"")); - - HRN_STORAGE_PUT_Z( - storageTest, "repo/test/src/module/test/shimTest.c", - "static void\n" - "testRun(void)\n" - "{\n" - " FUNCTION_HARNESS_VOID();\n" - "\n" - " if (testBegin(\"shims\"))\n" - " {\n" - " TEST_RESULT_INT(shimFunc_SHIMMED(), 777, \"shimFunc()\");\n" - " TEST_RESULT_INT(shimFunc(), 778, \"shimFunc()\");\n" - "\n" - " TEST_RESULT_INT(shimFunc2_SHIMMED(111, 112), 1000, \"shimFunc2()\");\n" - " TEST_RESULT_INT(shimFunc2(111, 112), 1002, \"shimFunc2()\");\n" - "\n" - " TEST_RESULT_INT(noShimFunc3(), 888, \"noShimFunc3()\");\n" - " }\n" - "\n" - " FUNCTION_HARNESS_RETURN_VOID();\n" - "}\n"); - - HRN_STORAGE_PUT_EMPTY(storageTest, TEST_PATH "/test/unit-3/uXX/cleanme.txt"); - - TEST_RESULT_VOID( - cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, STRDEF("invalid"), - STRDEF("test/shim"), 0, 1, logLevelDebug, true, NULL, architecture, true, -#ifdef DEBUG_COVERAGE - true, -#else - false, -#endif - true, true), - "new build"); - - // Older versions of ninja may error on a rebuild so a retry may occur - TEST_RESULT_LOG_EMPTY_OR_CONTAINS("WARN: build failed for unit"); - - storageUnit = storagePosixNewP(STRDEF(TEST_PATH "/test/unit-3/uXX")); - fileList = testStorageList(storageUnit); - - TEST_RESULT_STRLST_Z( - fileList, - "meson.build\n" - "meson_options.txt\n" - "src/common/stackTrace.c\n" - "test/common/shimInc.c.inc\n" - "test/src/common/shim.c\n" - "test/src/harness/error.c\n" - "test/src/harness/shim.c\n" - "test/src/harness/stackTrace.c\n" - "test.c\n", - "check files"); - - for (unsigned int fileIdx = 0; fileIdx < strLstSize(fileList); fileIdx++) - { - const String *const file = strLstGet(fileList, fileIdx); - - if (strEqZ(file, "meson.build")) - { - TEST_STORAGE_GET( - storageUnit, strZ(file), - zNewFmt( - "%s" - "add_global_arguments('-DHRN_FEATURE_ERROR', language : 'c')\n" - "add_global_arguments('-DHRN_FEATURE_STACKTRACE', language : 'c')\n" - "add_global_arguments('-DNDEBUG', language : 'c')\n" - "add_global_arguments('-DDEBUG_COVERAGE', language : 'c')\n" - "add_global_arguments('-DTEST_CONTAINER_REQUIRED', language : 'c')\n" - "\n" - MESON_COMMENT_BLOCK "\n" - "# Unit test\n" - MESON_COMMENT_BLOCK "\n" - "src_unit = files(\n" - " '../../../repo/src/common/type/stringStatic.c',\n" - " '../../../repo/src/common/debug.c',\n" - " 'test/src/harness/stackTrace.c',\n" - " '../../../repo/test/src/harness/noShim.c',\n" - " '../../../repo/test/src/harness/shim/sub.c',\n" - " '../../../repo/test/src/harness/test.c',\n" - " 'test.c',\n" - ")\n" - "\n" - "executable(\n" - " 'test-unit',\n" - " sources: src_unit,\n" - " c_args: [\n" - " '-O2',\n" -#ifdef DEBUG_COVERAGE - " '-pg',\n" - " '-no-pie',\n" - " ],\n" - " link_args: [\n" - " '-pg',\n" - " '-no-pie',\n" -#endif - " ],\n" - " include_directories:\n" - " include_directories(\n" - " '.',\n" - " '../../../repo/src',\n" - " '../../../repo/doc/src',\n" - " '../../../repo/test/src',\n" - " ),\n" - " dependencies: [\n" - " lib_backtrace,\n" - " lib_bz2,\n" - " lib_openssl,\n" - " lib_lz4,\n" - " lib_pq,\n" - " lib_ssh2,\n" - " lib_xml,\n" - " lib_yaml,\n" - " lib_z,\n" - " lib_zstd,\n" - " ],\n" - ")\n", - strZ(mesonBuildRoot))); - } - else if (strEqZ(file, "meson_options.txt")) - { - TEST_STORAGE_GET(storageUnit, strZ(file), mesonOption); - } - else if (strEqZ(file, "src/common/stackTrace.c") || strEqZ(file, "test/src/harness/stackTrace.c")) - { - // No test needed - } - else if (strEqZ(file, "test/src/harness/error.c")) - { - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(harnessErrorC)); - } - else if (strEqZ(file, "test/src/harness/shim.c")) - { - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(harnessShimC)); - } - else if (strEqZ(file, "test/src/common/shim.c")) - { - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(shimC)); - } - else if (strEqZ(file, "test/common/shimInc.c.inc")) - { - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(shimIncC)); - } - else if (strEqZ(file, "test.c")) - { - String *const testCDup = strCat(strNew(), testC); - - strReplace(testCDup, STRDEF("{[C_TEST_CONTAINER]}"), STRDEF("true")); - strReplace(testCDup, STRDEF("{[C_TEST_VM]}"), STRDEF("uXX")); - strReplace(testCDup, STRDEF("{[C_TEST_PG_VERSION]}"), STRDEF("invalid")); - strReplace(testCDup, STRDEF("{[C_TEST_LOG_EXPECT]}"), STRDEF("true")); -#ifdef DEBUG_COVERAGE - strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("// Debug test trace not enabled")); -#else - strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("#define DEBUG_TEST_TRACE")); -#endif - strReplace(testCDup, STRDEF("{[C_TEST_PATH_BUILD]}"), STRDEF(TEST_PATH "/test/unit-3/uXX/build")); -#ifdef DEBUG_COVERAGE - strReplace(testCDup, STRDEF("{[C_TEST_PROFILE]}"), STRDEF("true")); -#else - strReplace(testCDup, STRDEF("{[C_TEST_PROFILE]}"), STRDEF("false")); -#endif - strReplace(testCDup, STRDEF("{[C_TEST_PROJECT_EXE]}"), STRDEF(TEST_PATH "/test/build/uXX/src/pgbackrest")); - strReplace(testCDup, STRDEF("{[C_TEST_TZ]}"), STRDEF("// No timezone specified")); - - strReplace( - testCDup, - STRDEF("{[C_INCLUDE]}"), - STRDEF( - "#include \"test/src/harness/shim.c\"\n" - "#include \"test/src/harness/error.c\"\n" - "#include \"../../../repo/test/src/common/include.c\"\n" - "#include \"../../../repo/doc/src/command/build/build.c\"")); - strReplace( - testCDup, STRDEF("{[C_TEST_INCLUDE]}"), STRDEF("#include \"../../../repo/test/src/module/test/shimTest.c\"")); - strReplace( - testCDup, STRDEF("{[C_TEST_LIST]}"), - STRDEF( - "hrnAdd( 1, true);")); - - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(testCDup)); - } - else - THROW_FMT(TestError, "no test for '%s'", strZ(file)); - } - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("Run test/shim and build again to cleanup coverage data"); - - HRN_SYSTEM(TEST_PATH "/test/unit-3/uXX/build/test-unit"); - - TEST_RESULT_VOID( - cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, STRDEF("invalid"), - STRDEF("test/shim"), 0, 1, logLevelDebug, true, NULL, architecture, true, -#ifdef DEBUG_COVERAGE - true, -#else - false, -#endif - true, true), - "new build"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("Test real/all"); - - HRN_STORAGE_PUT_Z( - storageTest, "repo/test/src/module/real/allTest.c", - "static void\n" - "testRun(void)\n" - "{\n" - " FUNCTION_HARNESS_VOID();\n" - "\n" - " if (testBegin(\"all\"))\n" - " {\n" - " }\n" - "\n" - " FUNCTION_HARNESS_RETURN_VOID();\n" - "}\n"); - - TEST_RESULT_VOID( - cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, STRDEF("invalid"), - STRDEF("real/all"), 0, 1, logLevelDebug, true, STRDEF("America/New_York"), architecture, false, -#ifdef DEBUG_COVERAGE - true, -#else - false, -#endif - false, true), - "new build"); - - storageUnit = storagePosixNewP(STRDEF(TEST_PATH "/test/unit-3/none")); - fileList = testStorageList(storageUnit); - - TEST_RESULT_STRLST_Z( - fileList, - "meson.build\n" - "meson_options.txt\n" - "test/common/shimInc.c.inc\n" - "test/src/common/shim.c\n" - "test/src/harness/error.c\n" - "test/src/harness/shim.c\n" - "test.c\n", - "check files"); - - for (unsigned int fileIdx = 0; fileIdx < strLstSize(fileList); fileIdx++) - { - const String *const file = strLstGet(fileList, fileIdx); - - if (strEqZ(file, "meson.build")) - { - TEST_STORAGE_GET( - storageUnit, strZ(file), - zNewFmt( - "%s" - "add_global_arguments('-DHRN_FEATURE_ERROR', language : 'c')\n" - "add_global_arguments('-DHRN_FEATURE_STACKTRACE', language : 'c')\n" - "\n" - MESON_COMMENT_BLOCK "\n" - "# Unit test\n" - MESON_COMMENT_BLOCK "\n" - "src_unit = files(\n" - " '../../../repo/src/common/stackTrace.c',\n" - " '../../../repo/src/common/type/stringStatic.c',\n" - " '../../../repo/src/common/debug.c',\n" - " 'test/src/harness/error.c',\n" - " '../../../repo/test/src/harness/noShim.c',\n" - " 'test/src/harness/shim.c',\n" - " '../../../repo/test/src/harness/test.c',\n" - " 'test.c',\n" - ")\n" - "\n" - "executable(\n" - " 'test-unit',\n" - " sources: src_unit,\n" -#ifdef DEBUG_COVERAGE - " c_args: [\n" - " '-pg',\n" - " '-no-pie',\n" - " ],\n" - " link_args: [\n" - " '-pg',\n" - " '-no-pie',\n" - " ],\n" -#endif - " include_directories:\n" - " include_directories(\n" - " '.',\n" - " '../../../repo/src',\n" - " '../../../repo/doc/src',\n" - " '../../../repo/test/src',\n" - " ),\n" - " dependencies: [\n" - " lib_backtrace,\n" - " lib_bz2,\n" - " lib_openssl,\n" - " lib_lz4,\n" - " lib_pq,\n" - " lib_ssh2,\n" - " lib_xml,\n" - " lib_yaml,\n" - " lib_z,\n" - " lib_zstd,\n" - " ],\n" - ")\n", - strZ(mesonBuildRoot))); - } - else if (strEqZ(file, "meson_options.txt")) - { - TEST_STORAGE_GET(storageUnit, strZ(file), mesonOption); - } - else if (strEqZ(file, "src/common/stackTrace.c") || strEqZ(file, "test/src/harness/stackTrace.c")) - { - // No test needed - } - else if (strEqZ(file, "test/src/harness/error.c")) - { - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(harnessErrorC)); - } - else if (strEqZ(file, "test/src/harness/shim.c")) - { - String *const harnessShimCInt = strCat(strNew(), harnessShimC); - strReplace(harnessShimCInt, STRDEF("uXX"), STRDEF("none")); - - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(harnessShimCInt)); - } - else if (strEqZ(file, "test/src/common/shim.c")) - { - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(shimC)); - } - else if (strEqZ(file, "test/common/shimInc.c.inc")) - { - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(shimIncC)); - } - else if (strEqZ(file, "test.c")) - { - String *const testCDup = strCat(strNew(), testC); - - strReplace(testCDup, STRDEF("{[C_TEST_CONTAINER]}"), STRDEF("false")); - strReplace(testCDup, STRDEF("{[C_TEST_VM]}"), STRDEF("uXX")); - strReplace(testCDup, STRDEF("{[C_TEST_PG_VERSION]}"), STRDEF("invalid")); - strReplace(testCDup, STRDEF("{[C_TEST_LOG_EXPECT]}"), STRDEF("false")); -#ifdef DEBUG_COVERAGE - strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("// Debug test trace not enabled")); -#else - strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("#define DEBUG_TEST_TRACE")); -#endif - strReplace(testCDup, STRDEF("{[C_TEST_PATH_BUILD]}"), STRDEF(TEST_PATH "/test/unit-3/none/build")); -#ifdef DEBUG_COVERAGE - strReplace(testCDup, STRDEF("{[C_TEST_PROFILE]}"), STRDEF("true")); -#else - strReplace(testCDup, STRDEF("{[C_TEST_PROFILE]}"), STRDEF("false")); -#endif - strReplace(testCDup, STRDEF("{[C_TEST_PROJECT_EXE]}"), STRDEF(TEST_PATH "/test/build/uXX/src/pgbackrest")); - strReplace(testCDup, STRDEF("{[C_TEST_TZ]}"), STRDEF("hrnTzSet(\"America/New_York\");")); - - strReplace(testCDup, STRDEF("{[C_INCLUDE]}"), STRDEF("")); - strReplace( - testCDup, STRDEF("{[C_TEST_INCLUDE]}"), STRDEF("#include \"../../../repo/test/src/module/real/allTest.c\"")); - strReplace( - testCDup, STRDEF("{[C_TEST_LIST]}"), - STRDEF( - "hrnAdd( 1, true);")); - - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(testCDup)); - } - else - THROW_FMT(TestError, "no test for '%s'", strZ(file)); - } - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("Test performance/type"); - - strReplace( - mesonBuildRoot, STRDEF(" configuration.set('HAVE_LIBBACKTRACE'"), - STRDEF("# configuration.set('HAVE_LIBBACKTRACE'")); - - HRN_STORAGE_PUT_Z( - storageTest, "repo/test/src/module/performance/typeTest.c", - "static void\n" - "testRun(void)\n" - "{\n" - " FUNCTION_HARNESS_VOID();\n" - "\n" - " if (testBegin(\"type\"))\n" - " {\n" - " }\n" - "\n" - " FUNCTION_HARNESS_RETURN_VOID();\n" - "}\n"); - - HRN_SYSTEM("chmod 000 " TEST_PATH "/test/unit-3/uXX/build"); - - TEST_RESULT_VOID( - cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, STRDEF("invalid"), - STRDEF("performance/type"), 0, 1, logLevelDebug, true, STRDEF("America/New_York"), architecture, false, -#ifdef DEBUG_COVERAGE - true, -#else - false, -#endif - false, false), - "new build"); - - TEST_RESULT_LOG( - "P00 WARN: build failed for unit performance/type -- will retry: unable to list file info for path '" TEST_PATH - "/test/unit-3/uXX/build': [13] Permission denied"); - - storageUnit = storagePosixNewP(STRDEF(TEST_PATH "/test/unit-3/uXX")); - fileList = testStorageList(storageUnit); - - TEST_RESULT_STRLST_Z( - fileList, - "meson.build\n" - "meson_options.txt\n" - "src/common/stackTrace.c\n" - "test/common/shimInc.c.inc\n" - "test/src/common/shim.c\n" - "test/src/harness/error.c\n" - "test/src/harness/shim.c\n" - "test/src/harness/stackTrace.c\n" - "test.c\n", - "check files"); - - for (unsigned int fileIdx = 0; fileIdx < strLstSize(fileList); fileIdx++) - { - const String *const file = strLstGet(fileList, fileIdx); - - if (strEqZ(file, "meson.build")) - { - TEST_STORAGE_GET( - storageUnit, strZ(file), - zNewFmt( - "%s" - "add_global_arguments('-DHRN_FEATURE_ERROR', language : 'c')\n" - "add_global_arguments('-DHRN_FEATURE_STACKTRACE', language : 'c')\n" - "add_global_arguments('-DTEST_CONTAINER_REQUIRED', language : 'c')\n" - "\n" - MESON_COMMENT_BLOCK "\n" - "# Unit test\n" - MESON_COMMENT_BLOCK "\n" - "src_unit = files(\n" - " '../../../repo/src/common/type/stringStatic.c',\n" - " '../../../repo/src/common/debug.c',\n" - " 'test/src/harness/error.c',\n" - " 'test/src/harness/stackTrace.c',\n" - " '../../../repo/test/src/harness/noShim.c',\n" - " '../../../repo/test/src/harness/shim/sub.c',\n" - " 'test/src/harness/shim.c',\n" - " '../../../repo/test/src/harness/test.c',\n" - " 'test.c',\n" - ")\n" - "\n" - "executable(\n" - " 'test-unit',\n" - " sources: src_unit,\n" - " c_args: [\n" - " '-O2',\n" -#ifdef DEBUG_COVERAGE - " '-pg',\n" - " '-no-pie',\n" - " ],\n" - " link_args: [\n" - " '-pg',\n" - " '-no-pie',\n" -#endif - " ],\n" - " include_directories:\n" - " include_directories(\n" - " '.',\n" - " '../../../repo/src',\n" - " '../../../repo/doc/src',\n" - " '../../../repo/test/src',\n" - " ),\n" - " dependencies: [\n" - " lib_bz2,\n" - " lib_openssl,\n" - " lib_lz4,\n" - " lib_pq,\n" - " lib_ssh2,\n" - " lib_xml,\n" - " lib_yaml,\n" - " lib_z,\n" - " lib_zstd,\n" - " ],\n" - ")\n", - strZ(mesonBuildRoot))); - } - else if (strEqZ(file, "meson_options.txt")) - { - TEST_STORAGE_GET(storageUnit, strZ(file), mesonOption); - } - else if (strEqZ(file, "src/common/stackTrace.c") || strEqZ(file, "test/src/harness/stackTrace.c")) - { - // No test needed - } - else if (strEqZ(file, "test/src/harness/error.c")) - { - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(harnessErrorC)); - } - else if (strEqZ(file, "test/src/harness/shim.c")) - { - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(harnessShimC)); - } - else if (strEqZ(file, "test/src/common/shim.c")) - { - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(shimC)); - } - else if (strEqZ(file, "test/common/shimInc.c.inc")) - { - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(shimIncC)); - } - else if (strEqZ(file, "test.c")) - { - String *const testCDup = strCat(strNew(), testC); - - strReplace(testCDup, STRDEF("{[C_TEST_CONTAINER]}"), STRDEF("true")); - strReplace(testCDup, STRDEF("{[C_TEST_VM]}"), STRDEF("uXX")); - strReplace(testCDup, STRDEF("{[C_TEST_PG_VERSION]}"), STRDEF("invalid")); - strReplace(testCDup, STRDEF("{[C_TEST_LOG_EXPECT]}"), STRDEF("false")); - strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("// Debug test trace not enabled")); - strReplace(testCDup, STRDEF("{[C_TEST_PATH_BUILD]}"), STRDEF(TEST_PATH "/test/unit-3/uXX/build")); -#ifdef DEBUG_COVERAGE - strReplace(testCDup, STRDEF("{[C_TEST_PROFILE]}"), STRDEF("true")); -#else - strReplace(testCDup, STRDEF("{[C_TEST_PROFILE]}"), STRDEF("false")); -#endif - strReplace(testCDup, STRDEF("{[C_TEST_PROJECT_EXE]}"), STRDEF(TEST_PATH "/test/build/uXX/src/pgbackrest")); - strReplace(testCDup, STRDEF("{[C_TEST_TZ]}"), STRDEF("hrnTzSet(\"America/New_York\");")); - - strReplace(testCDup, STRDEF("{[C_INCLUDE]}"), STRDEF("")); - strReplace( - testCDup, STRDEF("{[C_TEST_INCLUDE]}"), - STRDEF("#include \"../../../repo/test/src/module/performance/typeTest.c\"")); - strReplace( - testCDup, STRDEF("{[C_TEST_LIST]}"), - STRDEF( - "hrnAdd( 1, true);")); - - TEST_STORAGE_GET(storageUnit, strZ(file), strZ(testCDup)); - } - else - THROW_FMT(TestError, "no test for '%s'", strZ(file)); - } - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("Test performance/type with profiling off for coverage"); - - TEST_RESULT_VOID( - cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, STRDEF("invalid"), - STRDEF("performance/type"), 0, 1, logLevelDebug, true, STRDEF("America/New_York"), STRDEF(TEST_ARCHITECTURE), false, - false, false, false), - "new build"); - - // Older versions of ninja may error on a rebuild so a retry may occur - TEST_RESULT_LOG_EMPTY_OR_CONTAINS("WARN: build failed for unit"); - - storageUnit = storagePosixNewP(STRDEF(TEST_PATH "/test/unit-3/uXX")); - - // Test one file to make sure profiling was not enabled - String *const testCDup = strCat(strNew(), testC); - - strReplace(testCDup, STRDEF("{[C_TEST_CONTAINER]}"), STRDEF("true")); - strReplace(testCDup, STRDEF("{[C_TEST_VM]}"), STRDEF("uXX")); - strReplace(testCDup, STRDEF("{[C_TEST_PG_VERSION]}"), STRDEF("invalid")); - strReplace(testCDup, STRDEF("{[C_TEST_LOG_EXPECT]}"), STRDEF("false")); - strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("// Debug test trace not enabled")); - strReplace(testCDup, STRDEF("{[C_TEST_PATH_BUILD]}"), STRDEF(TEST_PATH "/test/unit-3/uXX/build")); - strReplace(testCDup, STRDEF("{[C_TEST_PROFILE]}"), STRDEF("false")); - strReplace(testCDup, STRDEF("{[C_TEST_PROJECT_EXE]}"), STRDEF(TEST_PATH "/test/build/uXX/src/pgbackrest")); - strReplace(testCDup, STRDEF("{[C_TEST_TZ]}"), STRDEF("hrnTzSet(\"America/New_York\");")); - - strReplace(testCDup, STRDEF("{[C_INCLUDE]}"), STRDEF("")); - strReplace( - testCDup, STRDEF("{[C_TEST_INCLUDE]}"), - STRDEF("#include \"../../../repo/test/src/module/performance/typeTest.c\"")); - strReplace( - testCDup, STRDEF("{[C_TEST_LIST]}"), - STRDEF( - "hrnAdd( 1, true);")); - - TEST_STORAGE_GET(storageUnit, "test.c", strZ(testCDup)); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("Fatal test error"); - - HRN_SYSTEM("chmod 000 " TEST_PATH "/repo/meson.build"); - - TEST_ERROR( - cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, STRDEF("invalid"), - STRDEF("performance/type"), 0, 1, logLevelDebug, true, STRDEF("America/New_York"), architecture, false, false, - false, false), - FileOpenError, - "build failed for unit performance/type: unable to open file '" TEST_PATH "/repo/meson.build' for read: [13] Permission" - " denied"); - - TEST_RESULT_LOG( - "P00 WARN: build failed for unit performance/type -- will retry: unable to open file '" TEST_PATH "/repo/meson.build'" - " for read: [13] Permission denied"); - } - - FUNCTION_HARNESS_RETURN_VOID(); -} diff --git a/test/test.pl b/test/test.pl index 10a414193..534db0462 100755 --- a/test/test.pl +++ b/test/test.pl @@ -454,9 +454,9 @@ eval # Set the ccache path #------------------------------------------------------------------------------------------------------------------------------- - # A single cache is shared by the host and all containers so a file compiled in one is a hit in the others, e.g. - # test-pgbackrest is built both on the host and in the build container. ccache is safe for concurrent access so there is no - # need to keep a separate cache per vm or vm index. Keep the cache in the test path so it is removed by --clean. + # A single cache is shared by the host and all containers so a file compiled in one is a hit in the others, e.g. a module + # compiled for one unit test is a hit for every other unit test that includes it. ccache is safe for concurrent access so there + # is no need to keep a separate cache per vm or vm index. Keep the cache in the test path so it is removed by --clean. my $strCCachePath = "${strTestPath}/ccache"; if (!$oStorageTest->pathExists($strCCachePath)) @@ -502,8 +502,7 @@ eval ($bDryRun ? '' : " && \\\n${strBuildPath}/src/build-code config ${strBackRestBase}/src") . ($bDryRun ? '' : " && \\\n${strBuildPath}/src/build-code error ${strBackRestBase}/src") . ($bDryRun ? '' : " && \\\n${strBuildPath}/src/build-code postgres-version ${strBackRestBase}/src") . - " && \\\n${strBuildPath}/src/build-code postgres ${strBackRestBase}/src ${strRepoCachePath}" . - " && \\\nninja -C ${strBuildPath} test/src/test-pgbackrest 2>&1"; + " && \\\n${strBuildPath}/src/build-code postgres ${strBackRestBase}/src ${strRepoCachePath}"; if (!-e $strBuildNinja) { @@ -575,6 +574,7 @@ eval $strFile eq 'test/ci.pl' || $strFile eq 'test/smoke.py' || $strFile eq 'test/test.pl' || + $strFile eq 'test/test.py' || $strFile eq 'src/build/dist.sh' || $hManifest->{$strFile}{type} eq 'd') { @@ -609,7 +609,7 @@ eval # Determine if the C binary needs to be built foreach my $hTest (@{$oyTestRun}) { - # Unit build required for unit tests + # The build path is mounted into the test container so it must exist for unit tests even when nothing is built there if ($hTest->{&TEST_C}) { $bUnitRequired = true; @@ -649,24 +649,25 @@ eval &log(INFO, (!-e $strBuildNinja ? 'clean ' : '') . "build for ${strBuildVM} (${strBuildPath})"); - # Setup build if it does not exist - my $strBuildCommand = - "ninja -C ${strBuildPath}" . ($bBinRequired ? ' src/pgbackrest' : '') . - ($bUnitRequired ? ' test/src/test-pgbackrest' : '') . ' 2>&1'; + # Setup build if it does not exist. Unit tests are built from the repo copy by the test harness so there is + # nothing to build here for them, but the build path is still mounted into their container so it must exist. + my $strBuildCommand = $bBinRequired ? "ninja -C ${strBuildPath} src/pgbackrest 2>&1" : undef; if (!-e $strBuildNinja) { $strBuildCommand = "meson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug ${strBuildPath}" . - " ${strBackRestBase} && \\\n" . - $strBuildCommand; + " ${strBackRestBase}" . (defined($strBuildCommand) ? " && \\\n" . $strBuildCommand : ''); } # Build code - executeTest( - ($strBuildVM ne VM_NONE ? 'docker exec -i -u ' . TEST_USER . " test-build bash -c '" : '') . - $strBuildCommand . ($strBuildVM ne VM_NONE ? "'" : ''), - {bShowOutputAsync => $bLogDetail}); + if (defined($strBuildCommand)) + { + executeTest( + ($strBuildVM ne VM_NONE ? 'docker exec -i -u ' . TEST_USER . " test-build bash -c '" : '') . + $strBuildCommand . ($strBuildVM ne VM_NONE ? "'" : ''), + {bShowOutputAsync => $bLogDetail}); + } } } } @@ -807,8 +808,9 @@ eval } my $oExec = new pgBackRestTest::Common::ExecuteTest( - "${strBuildPath}/test/src/test-pgbackrest --log-level=warn --vm=${strVm} --repo-path=${strBackRestBase}" . - " --test-path=${strTestPath}" . ($bCoverageSummary ? ' --coverage-summary' : '') . " test${strModuleList}", + "python3 ${strBackRestBase}/test/test.py coverage --log-level=warn --vm=${strVm}" . + " --repo-path=${strBackRestBase} --test-path=${strTestPath}" . + ($bCoverageSummary ? ' --coverage-summary' : '') . $strModuleList, {bShowOutputAsync => true, bSuppressError => true}); $oExec->begin(); my $iResult = $oExec->end(); diff --git a/test/test.py b/test/test.py new file mode 100755 index 000000000..3766d059c --- /dev/null +++ b/test/test.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""pgBackRest Test Harness. + +Builds and runs the unit tests and reports coverage. Driven by test.pl, which selects the tests to run, manages the containers, and +calls this harness for each test. + +Exit status: 0 = success, 1 = the coverage command found modules missing coverage, greater = error. All output, including errors, +goes to stdout since the Perl test framework fails a test when anything is written to stderr.""" + +#################################################################################################################################### +import os +import signal +import sys +import time +import traceback + +# Send everything written to stderr to stdout instead. The Perl test framework reports only what a test writes to stdout, so an +# error raised before the handler in main() is installed, e.g. while importing below, would otherwise be lost entirely. +sys.stderr = sys.stdout + +# Do not cache bytecode. The harness runs from a copy of the repository that the linter then scans, so a __pycache__ written during +# import would be reported as an unexpected binary file. This must be set before the harness modules are imported below. +sys.dont_write_bytecode = True + +# The library lives beside this script. Insert it first so the harness modules are found before anything else on the path. +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "lib")) + +from common.error import EXIT_ERROR, TestError # noqa: E402 + +# PyYAML is the only thing these imports need outside the standard library, so a failure here is almost always a missing package. +# Coverage is also required but only by the test runner, which is a separate process and reports a missing one itself. +try: + from command.coverage.coverage import cmd_coverage # noqa: E402 + from command.lint.lint import cmd_lint # noqa: E402 + from command.test.unit import cmd_unit # noqa: E402 + from common.log import ERROR, INFO, log # noqa: E402 + from config.config import cfg_load, project_version # noqa: E402 +except ImportError as error: + print("unable to load the test harness: %s" % error) + print("HINT: PyYAML is required -- install python3-yaml (Debian), python3-pyyaml (RHEL), or py3-yaml (Alpine)") + print("HINT: rh8 needs python3.12 and python3.12-pyyaml, since the harness does not run on the platform python") + + sys.exit(EXIT_ERROR) + + +#################################################################################################################################### +def command_run(config): + """Run the requested command.""" + + if config.command == "unit": + cmd_unit(config) + elif config.command == "coverage": + return cmd_coverage(config) + else: + cmd_lint(config.repo_path) + + return 0 + + +#################################################################################################################################### +def main(): + """Main.""" + + # Die silently on SIGPIPE as C programs do, rather than raising when output is piped to a command that exits early + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + time_begin = time.time() + path_harness = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + # Load the configuration. Logging is not initialized until this succeeds so print any error directly. + try: + config = cfg_load(sys.argv[1:], path_harness) + except TestError as error: + print(error) + + return EXIT_ERROR + + log(INFO, "%s command begin %s: %s" % (config.command, project_version(path_harness), " ".join(sys.argv[1:]))) + + try: + result = command_run(config) + except TestError as error: + log(ERROR, error) + log(INFO, "%s command end: aborted with exception" % config.command) + + return error.status + except Exception: + # An unexpected exception is a harness bug, so show the traceback. It goes to stdout like everything else. + print(traceback.format_exc()) + + return EXIT_ERROR + + log(INFO, "%s command end: completed successfully (%ums)" % (config.command, int((time.time() - time_begin) * 1000))) + + return result + + +if __name__ == "__main__": + sys.exit(main())
LINEBRANCHCODE
" - -#define TEST_CVG_HTML_RPT_BRANCH_COVERED \ - "" - -#define TEST_CVG_HTML_RPT_BRANCH_UNCOVERED_POST \ - "" - -#define TEST_CVG_HTML_RPT_CODE_UNCOVERED \ - " " - -#define TEST_CVG_HTML_RPT_LINE_POST \ - "
...