diff --git a/.gitattributes b/.gitattributes index 9bfbcd932..9ba7b44b7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,12 +3,11 @@ *.c.inc linguist-language=C # The distribution tarball (git archive) ships only what is needed to build pgbackrest: exclude everything, then re-include the -# build inputs. src/build (code generation) is excluded because dist.sh ships the generated files instead. dist.sh also adds the -# generated docs, the .inc files, the dist marker, and a distribution README. The smoke test (test/smoke.py) and the meson.build that -# runs it are shipped so packagers can verify a build; the rest of test is excluded. +# build inputs. The code generation tooling in /build is excluded because dist.sh ships the generated files instead. dist.sh also +# adds the generated docs, the .inc files, the dist marker, and a distribution README. The smoke test (test/smoke.py) and the +# meson.build that runs it are shipped so packagers can verify a build; the rest of test is excluded. /* export-ignore /src -export-ignore -/src/build export-ignore /test -export-ignore /test/* export-ignore /test/meson.build -export-ignore diff --git a/.github/workflows/test-arch.yml b/.github/workflows/test-arch.yml index 405b36149..0ed0e583f 100644 --- a/.github/workflows/test-arch.yml +++ b/.github/workflows/test-arch.yml @@ -59,7 +59,7 @@ jobs: - name: Install run: | sudo apt-get update - sudo DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y sudo python3-yaml rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config make gcc ccache meson git liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev + sudo DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y sudo python3-yaml rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev pkg-config make gcc ccache meson git liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev docker run --privileged --rm tonistiigi/binfmt --install all - name: Login to GHCR diff --git a/.github/workflows/test-bsd.yml b/.github/workflows/test-bsd.yml index c4348a7f2..b4e89f4ca 100644 --- a/.github/workflows/test-bsd.yml +++ b/.github/workflows/test-bsd.yml @@ -13,8 +13,8 @@ jobs: macos: runs-on: macos-26 env: - LDFLAGS: -L/opt/homebrew/opt/openssl@3/lib -L/opt/homebrew/opt/libpq/lib -L/opt/homebrew/opt/libxml2/lib -L/opt/homebrew/opt/libyaml/lib - CPPFLAGS: -I/opt/homebrew/opt/openssl@3/include -I/opt/homebrew/opt/libpq/include -I/opt/homebrew/opt/libxml2/include/libxml2 -I/opt/homebrew/opt/libyaml/include + LDFLAGS: -L/opt/homebrew/opt/openssl@3/lib -L/opt/homebrew/opt/libpq/lib -L/opt/homebrew/opt/libxml2/lib + CPPFLAGS: -I/opt/homebrew/opt/openssl@3/include -I/opt/homebrew/opt/libpq/include -I/opt/homebrew/opt/libxml2/include/libxml2 PKG_CONFIG_PATH: /opt/homebrew/opt/libpq/lib/pkgconfig:/opt/homebrew/opt/openssl@3/lib/pkgconfig steps: @@ -39,7 +39,7 @@ jobs: esac done - brew install -q pkg-config openssl@3 libpq libxml2 libyaml meson + brew install -q pkg-config openssl@3 libpq libxml2 meson # PyYAML for the test harness. Homebrew has no formula for it and its python is externally managed, so this is the # supported way to add a package to it. @@ -138,7 +138,7 @@ jobs: # postgresql*-client provides the libpq to build against and postgresql*-server provides the tooling the smoke test needs # to create a cluster - sudo pkg install -y bash git ${pgServer%-server}-client ${pgServer} pkgconf libxml2 gmake libyaml rsync meson + sudo pkg install -y bash git ${pgServer%-server}-client ${pgServer} pkgconf libxml2 gmake rsync meson # PyYAML for the test harness. The package is py-pyyaml, renamed from py-yaml in 2024 so both names are # searched, and it is built only for the python flavors the ports tree selects, which is not necessarily the version the diff --git a/.github/workflows/test-dist.yml b/.github/workflows/test-dist.yml index 2c6d9c63a..b82144d30 100644 --- a/.github/workflows/test-dist.yml +++ b/.github/workflows/test-dist.yml @@ -33,7 +33,7 @@ jobs: run: | sudo apt-get update DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive sudo apt-get install -y meson gcc pkg-config \ - libssl-dev libxml2-dev libpq-dev libyaml-dev liblz4-dev libzstd-dev libbz2-dev zlib1g-dev libxml-checker-perl + libssl-dev libxml2-dev libpq-dev liblz4-dev libzstd-dev libbz2-dev zlib1g-dev libxml-checker-perl - name: Build Tarball run: | @@ -45,11 +45,10 @@ jobs: # tarball but not used by the build or the smoke test. pgbackrest/doc/release.pl --dist --no-exe - # Configure the build and build the code generator, which meson dist runs to produce the shipped generated files + # Configure the build, which meson dist runs the code generator from to produce the shipped generated files meson setup build pgbackrest - ninja -C build src/build-code - # Build the tarball. meson dist runs src/build/dist.sh to bake in the generated files, documentation, and dist marker. + # Build the tarball. meson dist runs build/dist.sh to bake in the generated files, documentation, and dist marker. # --no-tests skips building and testing the extracted tarball here since that is exactly what the build jobs below do, # each on its own distribution. meson dist -C build --formats gztar --no-tests diff --git a/.github/workflows/test-misc.yml b/.github/workflows/test-misc.yml index f6c684a06..c852f88da 100644 --- a/.github/workflows/test-misc.yml +++ b/.github/workflows/test-misc.yml @@ -22,7 +22,7 @@ jobs: - name: Install run: | sudo apt-get update - DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive sudo apt-get install -y zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config meson liblz4-dev libzstd-dev libbz2-dev + DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive sudo apt-get install -y zlib1g-dev libssl-dev libxml2-dev libpq-dev pkg-config meson liblz4-dev libzstd-dev libbz2-dev - name: Build run: | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9e7cb1834..2543276a8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,8 +32,7 @@ pgbackrest-dev => Install development tools ``` sudo apt-get install rsync git build-essential valgrind libssl-dev zlib1g-dev \ libxml2-dev libpq-dev pkg-config libxml-checker-perl liblz4-dev liblz4-tool \ - zstd libzstd-dev bzip2 libbz2-dev libyaml-dev ccache meson python3-yaml \ - python3-coverage + zstd libzstd-dev bzip2 libbz2-dev ccache meson python3-yaml python3-coverage ``` Some unit tests and all the integration tests require Docker. Running in containers allows us to simulate multiple hosts, test on different distributions and versions of PostgreSQL, and use sudo without affecting the host system. @@ -235,12 +234,12 @@ pgbackrest/test/test.py --dry-run P00 INFO: test command begin 2.60.0dev: --dry-run --no-log-timestamp P00 INFO: test begin on aarch64 - log level info P00 INFO: clean autogenerate code ---> P00 INFO: 104 tests selected +--> P00 INFO: 117 tests selected - P00 INFO: P1-T001/104 - vm=none, module=common, test=error - [filtered 101 lines of output] - P00 INFO: P1-T103/104 - vm=none, module=performance, test=type - P00 INFO: P1-T104/104 - vm=none, module=performance, test=storage + P00 INFO: P1-T001/117 - vm=none, module=common, test=error + [filtered 114 lines of output] + P00 INFO: P1-T116/117 - vm=none, module=performance, test=type + P00 INFO: P1-T117/117 - vm=none, module=performance, test=storage --> P00 INFO: DRY RUN COMPLETED SUCCESSFULLY P00 INFO: test command end: completed successfully ``` @@ -564,8 +563,8 @@ Options can be added to a command or multiple commands. Options can be configura To add an option, two files need be to be modified: -- `src/build/config/config.yaml` -- `src/build/help/help.xml` +- `build/config.yaml` +- `doc/xml/reference.xml` These files are discussed in the following sections along with how to verify the `help` command output. @@ -573,7 +572,7 @@ These files are discussed in the following sections along with how to verify the There are detailed comment blocks above each section that explain the rules for defining commands and options. Regarding options, there are two types: 1) command line only, and 2) configuration file. With the exception of secrets, all configuration file options can be passed on the command line. To configure an option for the configuration file, the `section:` key must be present. -The `option:` section is broken into sub-sections by a simple comment divider (e.g. `# Repository options`) under which the options are organized alphabetically by option name. To better explain this section, two hypothetical examples will be discussed. For more details, see [config.yaml](https://github.com/pgbackrest/pgbackrest/blob/main/src/build/config/config.yaml). +The `option:` section is broken into sub-sections by a simple comment divider (e.g. `# Repository options`) under which the options are organized alphabetically by option name. To better explain this section, two hypothetical examples will be discussed. For more details, see [config.yaml](https://github.com/pgbackrest/pgbackrest/blob/main/build/config.yaml). #### EXAMPLE 1 hypothetical command line only option ``` diff --git a/build/build.py b/build/build.py new file mode 100755 index 000000000..34377db7b --- /dev/null +++ b/build/build.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""pgBackRest Code Builder. + +Generates the C that would be tedious and error prone to maintain by hand: the option parse tables, the error list, the help, and the +PostgreSQL interfaces. Each is generated from a declaration in src/build, which is the definition of record for what it describes. + +A generated file is only written when its content changes, so a run that generates nothing new leaves every timestamp alone and does +not make the build rebuild what depends on it. + +Exit status: 0 = success, greater = error.""" + +#################################################################################################################################### +import os +import sys + +# Do not cache bytecode. The tool runs from the source tree, where a __pycache__ would show up as an unexpected binary file in the +# linter and in the distribution. This must be set before the library modules are imported below. +sys.dont_write_bytecode = True + +# The library lives beside this script. Insert it first so the build modules are found before anything else on the path. +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "lib")) + +from command.main import cfg_load, cmd_build # noqa: E402 +from common.error import ToolError # noqa: E402 + + +#################################################################################################################################### +def main(): + """Generate the code named on the command line.""" + + # The repository is the one this script is part of unless the caller names another, which the documentation does since it + # generates its own option parse tables and help from its own declarations + path_repo = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + try: + config = cfg_load(sys.argv[1:], path_repo) + cmd_build(config) + except ToolError as error: + print("ERROR: %s" % error) + + return error.status + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/build/config/config.yaml b/build/config.yaml similarity index 100% rename from src/build/config/config.yaml rename to build/config.yaml diff --git a/src/build/dist.sh b/build/dist.sh similarity index 67% rename from src/build/dist.sh rename to build/dist.sh index e47ee9381..37d5c358c 100755 --- a/src/build/dist.sh +++ b/build/dist.sh @@ -4,21 +4,15 @@ # the code files, copy the man pages and HTML documentation into doc, copy the distribution README, and write the dist marker that # tells the build to use the shipped generated files. # -# build-code reads its inputs from the source tree and writes the generated files into the distribution. The inputs come from the -# source rather than the distribution because src/build (the code generation sources) is excluded from the tarball. The read path -# is passed with a trailing component because build-code uses the parent directory of that argument as the repo root. +# The generator reads its inputs from the source tree and writes the generated files into the distribution, since the declarations it +# reads are excluded from the tarball. #################################################################################################################################### set -eu -build_code="$MESON_BUILD_ROOT/src/build-code" +build_code="$MESON_SOURCE_ROOT/build/build.py" -if [ ! -x "$build_code" ]; then - echo "dist: $build_code not found; run ninja before meson dist" >&2 - exit 1 -fi - -"$build_code" help "$MESON_SOURCE_ROOT/." "$MESON_DIST_ROOT" -"$build_code" postgres "$MESON_SOURCE_ROOT/." "$MESON_DIST_ROOT" +"$build_code" help --repo-path="$MESON_SOURCE_ROOT" --build-path="$MESON_DIST_ROOT" +"$build_code" postgres --repo-path="$MESON_SOURCE_ROOT" --build-path="$MESON_DIST_ROOT" # Copy the man pages and HTML documentation generated by doc/release.pl --dist into doc/man and doc/html mkdir -p "$MESON_DIST_ROOT/doc" diff --git a/src/build/error/error.yaml b/build/error.yaml similarity index 100% rename from src/build/error/error.yaml rename to build/error.yaml diff --git a/build/lib/command/main.py b/build/lib/command/main.py new file mode 100644 index 000000000..58902b3bf --- /dev/null +++ b/build/lib/command/main.py @@ -0,0 +1,120 @@ +"""Build Command. + +Parses the command line and runs the requested generator. Each generator reads a declaration from the repository and writes the C it +describes into the build path, which is the repository itself for the files that are committed and the build directory for the files +meson generates as it builds. + +The documentation generates its own option parse tables and help from its own declarations, so it runs these same generators with the +repository set to the documentation directory.""" + +#################################################################################################################################### +import argparse +import os + +from config.parse import bld_cfg_parse +from config.render import bld_cfg_render +from error.parse import bld_err_parse +from error.render import bld_err_render +from help.parse import bld_hlp_parse +from help.render import bld_hlp_render, bld_hlp_render_data + +# The help declaration, which lives with the documentation because that is what most of it is for +_PATH_HELP = "doc/xml/reference.xml" +from postgres.parse import bld_pg_parse +from postgres.render import bld_pg_render, bld_pg_version_render + + +#################################################################################################################################### +def _path_help(config): + """Path of the help declaration.""" + + return os.path.join(config.repo_path, _PATH_HELP) + + +#################################################################################################################################### +def _build_config(config): + """Generate the command and option configuration.""" + + bld_cfg_render(config.build_path, bld_cfg_parse(config.repo_path), True) + + +#################################################################################################################################### +def _build_error(config): + """Generate the error types.""" + + bld_err_render(config.build_path, bld_err_parse(config.repo_path)) + + +#################################################################################################################################### +def _build_help(config): + """Generate the help the binary carries for itself.""" + + bld_cfg = bld_cfg_parse(config.repo_path) + + bld_hlp_render(config.build_path, bld_cfg, bld_hlp_parse(_path_help(config), bld_cfg, False)) + + +#################################################################################################################################### +def _build_help_data(config): + """Generate the help as raw data, which is what the help unit test loads.""" + + bld_cfg = bld_cfg_parse(config.repo_path) + + bld_hlp_render_data(config.build_path, bld_cfg, bld_hlp_parse(_path_help(config), bld_cfg, False)) + + +#################################################################################################################################### +def _build_postgres(config): + """Generate the PostgreSQL interfaces.""" + + bld_pg_render(config.build_path, bld_pg_parse(config.repo_path)) + + +#################################################################################################################################### +def _build_postgres_version(config): + """Generate the PostgreSQL version constants.""" + + bld_pg_version_render(config.build_path, bld_pg_parse(config.repo_path)) + + +#################################################################################################################################### +# Generators, in the order they are listed in the help +_COMMAND_LIST = { + "config": _build_config, + "error": _build_error, + "help": _build_help, + "help-data": _build_help_data, + "postgres": _build_postgres, + "postgres-version": _build_postgres_version, +} + + +#################################################################################################################################### +def cfg_load(arg_list, path_repo): + """Parse the command line.""" + + parser = argparse.ArgumentParser(prog="build.py", description="Generate code for pgBackRest.") + parser.add_argument("command", choices=_COMMAND_LIST, metavar="COMMAND", help="what to generate: %s" % ", ".join(_COMMAND_LIST)) + parser.add_argument("--repo-path", default=path_repo, metavar="PATH", help="path the declarations are read from") + parser.add_argument("--build-path", metavar="PATH", help="path the generated code is written to, defaults to the repository") + + result = parser.parse_args(arg_list) + + # A relative path is made absolute so it does not depend on where the tool was run from, which meson does not promise + result.repo_path = os.path.abspath(result.repo_path) + + # The generated code goes into the repository unless the caller sends it somewhere else, which meson does for the files it + # generates as it builds rather than the files that are committed + if result.build_path is None: + result.build_path = result.repo_path + else: + result.build_path = os.path.abspath(result.build_path) + + return result + + +#################################################################################################################################### +def cmd_build(config): + """Run the requested generator.""" + + _COMMAND_LIST[config.command](config) diff --git a/test/lib/common/error.py b/build/lib/common/error.py similarity index 60% rename from test/lib/common/error.py rename to build/lib/common/error.py index 31ee50e7d..e5a616472 100644 --- a/test/lib/common/error.py +++ b/build/lib/common/error.py @@ -1,7 +1,7 @@ """Error Handling. -The harness reports failures with TestError, which carries the exit status to return. Exit status mirrors the C harness: 0 is -success, 1 means a top-level coverage check found modules missing coverage, and anything greater is an error.""" +Each tool reports failures with ToolError, which carries the exit status to return. Exit status mirrors the C: 0 is success, 1 means a +top-level coverage check found modules missing coverage, and anything greater is an error.""" #################################################################################################################################### # Exit status for an error. Distinct from 1, which a coverage command uses to report incomplete coverage rather than a failure. @@ -9,7 +9,7 @@ EXIT_ERROR = 2 #################################################################################################################################### -class TestError(Exception): +class ToolError(Exception): """A harness error with the exit status to return.""" def __init__(self, message, status=EXIT_ERROR): @@ -19,10 +19,10 @@ class TestError(Exception): #################################################################################################################################### def check(condition, message): - """Raise TestError when a condition does not hold. + """Raise ToolError when a condition does not hold. - Used for invariants the harness itself must maintain, i.e. a failure means the harness or the definitions it reads are wrong - rather than the code under test.""" + Used for invariants a tool itself must maintain, i.e. a failure means the tool or the definitions it reads are wrong rather than + the code it is working on.""" if not condition: - raise TestError(message) + raise ToolError(message) diff --git a/test/lib/common/log.py b/build/lib/common/log.py similarity index 100% rename from test/lib/common/log.py rename to build/lib/common/log.py diff --git a/build/lib/common/pack.py b/build/lib/common/pack.py new file mode 100644 index 000000000..a4aad8e09 --- /dev/null +++ b/build/lib/common/pack.py @@ -0,0 +1,171 @@ +"""Pack Writer. + +Writes the compact binary format the C reads a pack with, which is how the help is shipped inside the binary. This is the write side +only and must agree with the reader in src/common/type/pack.c, since what it writes is read back by the C. + +A field is a one byte tag, which holds the type and the value, followed by the gap since the last field when it is too wide for the +tag. A field left null is not written at all -- the gap in the ids is what says it was there -- which is what makes the format compact +for data that is mostly defaults.""" + +#################################################################################################################################### +from common.error import check + +# Field types, which are the same numbers the reader maps back to types. Only the types the help needs are here, all of which fit in +# the four bits the tag has for a type. +_TYPE_ARRAY = 1 +_TYPE_BOOL = 2 +_TYPE_OBJ = 5 +_TYPE_STR = 7 + +# Types whose value is one bit in the tag, i.e. the value is there or it is not +_TYPE_SINGLE_BIT = (_TYPE_BOOL, _TYPE_STR) + + +#################################################################################################################################### +class _Tag: + """Where the ids are up to inside one container, since a container numbers its fields from one.""" + + def __init__(self, type): + self.type = type + self.id_last = 0 + self.null_total = 0 + + +#################################################################################################################################### +class PackWrite: + """Write a pack.""" + + def __init__(self): + self.buffer = bytearray() + self._tag_stack = [_Tag(None)] + + ################################################################################################################################ + def _u64(self, value): + """Write an integer base-128, seven bits at a time, with the high bit set on every byte but the last.""" + + while value >= 0x80: + self.buffer.append((value & 0x7F) | 0x80) + value >>= 7 + + self.buffer.append(value) + + ################################################################################################################################ + def _tag(self, type, id, value): + """Write a field tag, i.e. the type, the gap since the last field, and as much of the value as fits.""" + + tag_top = self._tag_stack[-1] + + # An id that is not given follows the last one, counting the fields that were left null + if id == 0: + id = tag_top.id_last + tag_top.null_total + 1 + else: + check(id > tag_top.id_last, "field id must be greater than last id") + + tag_top.null_total = 0 + + # The gap never needs to record the one it is always at least, so it is the difference less one + tag_id = id - tag_top.id_last - 1 + tag = type << 4 + + # A single bit value fits the tag entirely, so nothing of it is left to write after the tag + if type in _TYPE_SINGLE_BIT: + tag |= value << 3 + + tag |= tag_id & 0x3 + tag_id >>= 2 + + if tag_id > 0: + tag |= 0x4 + else: + check(value == 0, "no value expected") + + tag |= tag_id & 0x7 + tag_id >>= 3 + + if tag_id > 0: + tag |= 0x8 + + self.buffer.append(tag) + + if tag_id > 0: + self._u64(tag_id) + + tag_top.id_last = id + + ################################################################################################################################ + def null_write(self): + """Leave a field null, which writes nothing and leaves a gap in the ids.""" + + self._tag_stack[-1].null_total += 1 + + ################################################################################################################################ + def bool_write(self, value, id=0, default_write=False): + """Write a boolean, which is left null when it is false unless writing it is forced.""" + + if not default_write and value is False: + self.null_write() + + return + + self._tag(_TYPE_BOOL, id, 1 if value else 0) + + ################################################################################################################################ + def str_write(self, value, id=0): + """Write a string, which is left null when there is no string at all. + + An empty string is written, since the tag says whether there is a string rather than how long it is.""" + + if value is None: + self.null_write() + + return + + data = value.encode() + + self._tag(_TYPE_STR, id, 1 if len(data) > 0 else 0) + + if len(data) > 0: + self._u64(len(data)) + self.buffer.extend(data) + + ################################################################################################################################ + def array_begin(self, id=0): + """Begin an array, which numbers its fields from one.""" + + self._tag(_TYPE_ARRAY, id, 0) + self._tag_stack.append(_Tag(_TYPE_ARRAY)) + + ################################################################################################################################ + def array_end(self): + """End an array.""" + + check(self._tag_stack[-1].type == _TYPE_ARRAY, "not in array") + + self._u64(0) + self._tag_stack.pop() + + ################################################################################################################################ + def obj_begin(self, id=0): + """Begin an object, which numbers its fields from one.""" + + self._tag(_TYPE_OBJ, id, 0) + self._tag_stack.append(_Tag(_TYPE_OBJ)) + + ################################################################################################################################ + def obj_end(self): + """End an object.""" + + check(self._tag_stack[-1].type == _TYPE_OBJ, "not in object") + + self._u64(0) + self._tag_stack.pop() + + ################################################################################################################################ + def end(self): + """End the pack and return what was written.""" + + check(len(self._tag_stack) == 1, "container not ended") + + self._u64(0) + + return bytes(self.buffer) diff --git a/build/lib/common/render.py b/build/lib/common/render.py new file mode 100644 index 000000000..d70fcdce3 --- /dev/null +++ b/build/lib/common/render.py @@ -0,0 +1,76 @@ +"""Render Helpers. + +Naming and formatting shared by everything that renders C, so the generated code looks the same whichever generator wrote it.""" + +#################################################################################################################################### +from common.string_id import STRING_ID_SEQ_NONE, str_id_from_z, str_id_seq + +# Width every line of generated code is formatted to, which is the width the project uses everywhere +LINE_LENGTH = 132 + +# Block comment delimiters, e.g. /****...****/, and the separator that divides sections inside a function +COMMENT_BLOCK_BEGIN = "/" + "*" * (LINE_LENGTH - 1) +COMMENT_BLOCK_END = "*" * (LINE_LENGTH - 2) + "*/" +COMMENT_SEPARATOR = " // " + "-" * (LINE_LENGTH - len(" // ")) + +# Column a #define value is aligned at +_DEFINE_COLUMN = 68 + + +#################################################################################################################################### +def bld_define(define, value): + """Format a #define with the value aligned in a column, e.g. "#define TEST_STRIDSTRID5(...)".""" + + return "#define %s%*s%s" % (define, _DEFINE_COLUMN - len("#define ") - len(define), "", value) + + +#################################################################################################################################### +def bld_comment_block(comment): + """Format a block comment, which is how a section of generated code is titled.""" + + return "%s\n%s\n%s\n" % (COMMENT_BLOCK_BEGIN, comment, COMMENT_BLOCK_END) + + +#################################################################################################################################### +def bld_header(module, description): + """Format the header every generated file begins with, which says not to edit the file and what to run instead.""" + + return bld_comment_block("%s\n\nAutomatically generated by 'build.py %s' -- do not modify directly." % (description, module)) + + +#################################################################################################################################### +def bld_enum(prefix, value): + """Convert a dashed value to an enum name, e.g. "type-convert" becomes "typeConvert". + + When a prefix is given it is prepended and the first letter of the value is upper-cased, e.g. bld_enum("logLevel", "info") + becomes "logLevelInfo".""" + + result = "" + upper = False + + if prefix is not None: + result = prefix + upper = True + + for char in value: + # A dash is removed and the letter that follows it is upper-cased + if char == "-": + upper = True + continue + + result += char.upper() if upper else char + upper = False + + return result + + +#################################################################################################################################### +def bld_str_id_seq(value, sequence=STRING_ID_SEQ_NONE): + """Render the StringId macro for a value, e.g. STRID5("test", 0x2a7250), which is the form the source is expected to contain.""" + + str_id = str_id_from_z(value, sequence) + + if sequence is STRING_ID_SEQ_NONE: + return 'STRID%u("%s", 0x%x)' % ((str_id & 1) + 5, value, str_id) + + return 'STRID%uS("%s", %u, 0x%x)' % ((str_id & 1) + 5, value, str_id_seq(str_id), str_id) diff --git a/test/lib/common/storage.py b/build/lib/common/storage.py similarity index 82% rename from test/lib/common/storage.py rename to build/lib/common/storage.py index ffd57d775..9850ea114 100644 --- a/test/lib/common/storage.py +++ b/build/lib/common/storage.py @@ -1,13 +1,13 @@ """File System Helpers. Thin wrappers over the standard library that report the affected path in the error, since a bare OSError from deep in a generated -build says nothing about which file the harness was working on.""" +build says nothing about which file the tool was working on.""" #################################################################################################################################### import os import re -from common.error import TestError +from common.error import ToolError #################################################################################################################################### @@ -23,22 +23,22 @@ def file_read(path, ignore_missing=False): if ignore_missing: return None - raise TestError("unable to open file '%s' for read" % path) + raise ToolError("unable to open file '%s' for read" % path) except OSError as error: - raise TestError("unable to open file '%s' for read: %s" % (path, error.strerror)) + raise ToolError("unable to open file '%s' for read: %s" % (path, error.strerror)) #################################################################################################################################### def file_write(path, content): - """Write a file as text, creating parent paths as needed.""" + """Write a file as text, or as bytes when that is what was given, creating parent paths as needed.""" path_create(os.path.dirname(path)) try: - with open(path, "w") as file: + with open(path, "wb" if isinstance(content, bytes) else "w") as file: file.write(content) except OSError as error: - raise TestError("unable to open file '%s' for write: %s" % (path, error.strerror)) + raise ToolError("unable to open file '%s' for write: %s" % (path, error.strerror)) #################################################################################################################################### @@ -61,9 +61,9 @@ def file_remove(path, error_on_missing=False): os.remove(path) except FileNotFoundError: if error_on_missing: - raise TestError("unable to remove missing file '%s'" % path) + raise ToolError("unable to remove missing file '%s'" % path) except OSError as error: - raise TestError("unable to remove file '%s': %s" % (path, error.strerror)) + raise ToolError("unable to remove file '%s': %s" % (path, error.strerror)) #################################################################################################################################### @@ -78,7 +78,7 @@ def path_create(path, mode=0o750): try: os.makedirs(path, mode=mode, exist_ok=True) except OSError as error: - raise TestError("unable to create path '%s': %s" % (path, error.strerror)) + raise ToolError("unable to create path '%s': %s" % (path, error.strerror)) #################################################################################################################################### @@ -91,11 +91,11 @@ def path_list(path, expression=None, error_on_missing=False): result = sorted(os.listdir(path)) except FileNotFoundError: if error_on_missing: - raise TestError("unable to list missing path '%s'" % path) + raise ToolError("unable to list missing path '%s'" % path) return [] except OSError as error: - raise TestError("unable to list path '%s': %s" % (path, error.strerror)) + raise ToolError("unable to list path '%s': %s" % (path, error.strerror)) if expression is not None: regexp = re.compile(expression) diff --git a/test/lib/common/string_id.py b/build/lib/common/string_id.py similarity index 95% rename from test/lib/common/string_id.py rename to build/lib/common/string_id.py index 467e087d8..6bd572220 100644 --- a/test/lib/common/string_id.py +++ b/build/lib/common/string_id.py @@ -4,7 +4,7 @@ Port of the encoding in src/common/type/stringId.c. The maps there are written o character ranges they encode, which is the same mapping.""" #################################################################################################################################### -from common.error import TestError +from common.error import ToolError # Maximum characters that can be encoded with each number of bits STRID5_MAX = 12 @@ -79,7 +79,7 @@ def str_id_from_z(value, sequence=STRING_ID_SEQ_NONE): """Encode a string as a StringId, adding a sequence when one is specified.""" if value == "": - raise TestError("StringId value may not be empty") + raise ToolError("StringId value may not be empty") result = _str_id_bit_from_z(_BIT5, value) @@ -89,7 +89,7 @@ def str_id_from_z(value, sequence=STRING_ID_SEQ_NONE): # Error when 6-bit encoding also fails if result == 0: - raise TestError("'%s' contains invalid characters" % value) + raise ToolError("'%s' contains invalid characters" % value) if sequence is not STRING_ID_SEQ_NONE: # If the sequence fits in the header diff --git a/build/lib/common/xml.py b/build/lib/common/xml.py new file mode 100644 index 000000000..32fdafc45 --- /dev/null +++ b/build/lib/common/xml.py @@ -0,0 +1,206 @@ +"""Xml Handler. + +Thin helpers over the standard library element tree, shaped the way the documentation reads and writes xml. + +Text is the awkward part. A document like the user guide is mixed content -- text and markup interleaved in one paragraph -- which the +element tree stores as the text before a node's first child plus the text trailing each child. So adding text to a node means appending +to whichever of those came last, and reading a node's text means gathering all of it. Both are done here so nothing else has to know. + +Comments never appear in a parsed document, since the parser drops them, so nothing here has to skip them.""" + +#################################################################################################################################### +import copy +import re +import xml.etree.ElementTree as etree + +from common.error import ToolError + + +#################################################################################################################################### +class XmlDocument: + """An xml document, i.e. a root node plus the declarations that go above it.""" + + def __init__(self, root, dtd_name=None, dtd_file=None): + self.root = root + self.dtd_name = dtd_name # Document type, if any + self.dtd_file = dtd_file # File the document type is declared in + + ################################################################################################################################ + def render(self): + """Render the document as the xml text of it.""" + + result = '\n' + + if self.dtd_name is not None: + result += '\n' % (self.dtd_name, self.dtd_file) + + return result + etree.tostring(self.root, encoding="unicode") + "\n" + + +# Document type a document declares, which is kept so a document that is read and written again declares the same +_DTD_EXP = re.compile(r"""""") + + +#################################################################################################################################### +def xml_document_new(root_name, dtd_name=None, dtd_file=None): + """Build an empty document.""" + + return XmlDocument(etree.Element(root_name), dtd_name, dtd_file) + + +#################################################################################################################################### +def xml_document_parse(content, path): + """Parse a document, keeping the document type it declares.""" + + match = _DTD_EXP.search(content) + + return XmlDocument(xml_parse(content, path), *(match.groups() if match is not None else (None, None))) + + +#################################################################################################################################### +def xml_parse(content, path): + """Parse xml, naming the file in any error since the message is otherwise only a line number.""" + + try: + return etree.fromstring(content) + except etree.ParseError as error: + raise ToolError("unable to parse '%s': %s" % (path, error)) + + +#################################################################################################################################### +def xml_node_attribute(node, name, error_on_missing=False): + """Attribute of a node, or None when it is not there.""" + + result = node.get(name) + + if result is None and error_on_missing: + raise ToolError("unable to find attribute '%s' in node '%s'" % (name, node.tag)) + + return result + + +#################################################################################################################################### +def xml_node_attribute_set(node, name, value): + """Set an attribute of a node.""" + + node.set(name, value) + + +#################################################################################################################################### +def xml_node_attribute_remove(node, name): + """Remove an attribute of a node, which is not an error when it is not there.""" + + node.attrib.pop(name, None) + + +#################################################################################################################################### +def xml_node_child(node, name, error_on_missing=False): + """First child of a node with a name, or None when there is none.""" + + result = node.find(name) + + if result is None and error_on_missing: + raise ToolError("unable to find child '%s' in node '%s'" % (name, node.tag)) + + return result + + +#################################################################################################################################### +def xml_node_child_list(node, name): + """Every child of a node with a name.""" + + return node.findall(name) + + +#################################################################################################################################### +def xml_node_add(node, name): + """Add a child to a node.""" + + return etree.SubElement(node, name) + + +#################################################################################################################################### +def xml_node_dup(node): + """Copy a node and everything under it, so what is copied can be changed without changing what it came from.""" + + return copy.deepcopy(node) + + +#################################################################################################################################### +def xml_node_content(node): + """All the text under a node, with the markup dropped.""" + + return "".join(node.itertext()) + + +#################################################################################################################################### +def xml_node_content_add(node, content): + """Add text to the end of what a node holds. + + Text is added rather than set because a title is built from text and markup in turn, e.g. "Backup Command (" then an id then ")". + """ + + if len(node) == 0: + node.text = (node.text or "") + content + else: + node[-1].tail = (node[-1].tail or "") + content + + +#################################################################################################################################### +def xml_node_text_set(node, content): + """Replace everything a node holds with text.""" + + for child in list(node): + node.remove(child) + + node.text = content + + +#################################################################################################################################### +def xml_node_child_add(node, source): + """Add a copy of everything a source node holds to another node.""" + + xml_node_content_add(node, source.text or "") + + for child in source: + added = xml_node_add(node, child.tag) + + for name, value in child.attrib.items(): + xml_node_attribute_set(added, name, value) + + xml_node_child_add(added, child) + xml_node_content_add(node, child.tail or "") + + +#################################################################################################################################### +def xml_node_child_replace(parent, node, source): + """Replace a node with a copy of everything a source node holds. + + The parent is passed because a node does not know what holds it, and what replaces it takes its place rather than going at the + end.""" + + index = list(parent).index(node) + + # Build the replacement in a node of its own so the copy can be made with the same code that adds children anywhere else + replace = etree.Element(node.tag) + xml_node_child_add(replace, source) + + # Whatever trailed the node goes after whatever now ends the replacement + tail = node.tail or "" + parent.remove(node) + + for offset, child in enumerate(list(replace)): + parent.insert(index + offset, child) + + # Text the replacement begins with belongs to what precedes the node, i.e. the parent or the child before it + if index == 0: + parent.text = (parent.text or "") + replace.text + else: + parent[index - 1].tail = (parent[index - 1].tail or "") + replace.text + + if len(replace) > 0: + parent[index + len(replace) - 1].tail = (parent[index + len(replace) - 1].tail or "") + tail + elif index == 0: + parent.text = (parent.text or "") + tail + else: + parent[index - 1].tail = (parent[index - 1].tail or "") + tail diff --git a/build/lib/common/yaml.py b/build/lib/common/yaml.py new file mode 100644 index 000000000..38408c01d --- /dev/null +++ b/build/lib/common/yaml.py @@ -0,0 +1,77 @@ +"""Yaml Handler. + +Loads a declaration the way the generators need to read it, which is not the way yaml is usually loaded: + +- Every scalar stays the text it was written as. Yaml would otherwise type a scalar by how it looks, so a version written as 9.6 would + arrive as a float and render as 9.6000000000000005, and a code written as 011 would arrive as the octal 9. + +- A mapping keeps its pairs rather than becoming a dict, because a key may be repeated and the repetition is meaningful. The option + command list uses it to add several roles or exclude several commands, and a dict would silently keep only the last one. + +- A mapping keeps the order it was written in, which yaml guarantees for sequences only. Order matters because it decides the order of + the generated code, so the loader is what makes the declaration order the record.""" + +#################################################################################################################################### +import yaml + +from common.error import ToolError, check + + +#################################################################################################################################### +class YamlMap: + """A yaml mapping, as the list of key/value pairs it was written as.""" + + def __init__(self, pair_list): + self.pair_list = pair_list + + def __iter__(self): + return iter(self.pair_list) + + def __len__(self): + return len(self.pair_list) + + +#################################################################################################################################### +class _Loader(yaml.SafeLoader): + """A safe loader with no implicit typing and no dicts.""" + + +#################################################################################################################################### +def _construct_map(loader, node): + """Construct a mapping as its pairs, deeply, so a nested collection is complete rather than left to be filled in later.""" + + return YamlMap( + [(loader.construct_object(key, deep=True), loader.construct_object(value, deep=True)) for key, value in node.value] + ) + + +# Removing the implicit resolvers leaves plain scalars with the default string tag. An explicitly tagged scalar, e.g. !!int 1, is +# still typed, and there is none of that in the declarations. +_Loader.yaml_implicit_resolvers = {} +_Loader.add_constructor("tag:yaml.org,2002:map", _construct_map) + + +#################################################################################################################################### +def yaml_load(content, path): + """Load yaml from text, naming the file in any error since the message is otherwise only a line number.""" + + try: + return yaml.load(content, Loader=_Loader) + except yaml.YAMLError as error: + raise ToolError("unable to parse '%s': %s" % (path, error)) + + +#################################################################################################################################### +def yaml_bool(value, name): + """Parse a boolean, which must be written as true or false rather than any of the other spellings yaml allows.""" + + check(value in ("true", "false"), "invalid boolean '%s' for %s" % (value, name)) + + return value == "true" + + +#################################################################################################################################### +def yaml_map_empty(value, name): + """Check that a mapping is empty, i.e. the key is a name in a list rather than something with a definition.""" + + check(isinstance(value, YamlMap) and len(value) == 0, "%s must be an empty map" % name) diff --git a/build/lib/config/common.py b/build/lib/config/common.py new file mode 100644 index 000000000..0d6e9e4b0 --- /dev/null +++ b/build/lib/config/common.py @@ -0,0 +1,74 @@ +"""Config Value Parsing. + +Size and time options are written the way a person would write them, e.g. 1GiB or 30s, so the generated rules need the value they +mean in order to sort them and to compare a value against a range. This is a port of the parsing the C does at run time and must agree +with it, since a value the C parses differently would be checked against the wrong range. + +Only the parsing the generated code needs is here. The C also parses these at run time from user input, where the errors matter.""" + +#################################################################################################################################### +import re + +from common.error import ToolError + +# Size qualifiers, i.e. what the letter on the end of a size multiplies the number by +_SIZE_QUALIFIER = { + "b": 1, + "k": 1024, + "m": 1024 * 1024, + "g": 1024 * 1024 * 1024, + "t": 1024 * 1024 * 1024 * 1024, + "p": 1024 * 1024 * 1024 * 1024 * 1024, +} + +_SIZE_EXP = re.compile(r"^[0-9]+(kib|kb|k|mib|mb|m|gib|gb|g|tib|tb|t|pib|pb|p|b)*$") + +# Time qualifiers, in milliseconds, which is the unit the C keeps a time option in +_TIME_QUALIFIER = { + "ms": 1, + "s": 1000, + "m": 60 * 1000, + "h": 60 * 60 * 1000, + "d": 24 * 60 * 60 * 1000, + "w": 7 * 24 * 60 * 60 * 1000, +} + +# A time with no qualifier is in seconds +_TIME_MULTIPLIER_DEFAULT = 1000 + + +#################################################################################################################################### +def cfg_parse_size(value): + """The number of bytes a size value means, e.g. "1GiB" is 1073741824.""" + + value_lower = value.lower() + + if _SIZE_EXP.match(value_lower) is None: + raise ToolError("value '%s' is not valid" % value) + + # Find the qualifier, which is the first letter of whatever follows the number. A trailing b may be the qualifier itself or the + # bytes of a longer qualifier, e.g. the b in kb or in kib. + number = value_lower.rstrip("abcdefghijklmnopqrstuvwxyz") + qualifier = value_lower[len(number) : len(number) + 1] + + return int(number) * (_SIZE_QUALIFIER[qualifier] if qualifier != "" else 1) + + +#################################################################################################################################### +def cfg_parse_time(value): + """The number of milliseconds a time value means, e.g. "30s" is 30000 and "100ms" is 100.""" + + # The qualifier is what follows the number, which is ms or a single letter or nothing at all + number = value.rstrip("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + qualifier = value[len(number) :].lower() + + if qualifier != "" and qualifier not in _TIME_QUALIFIER: + raise ToolError("value '%s' is not valid" % value) + + if number == "": + raise ToolError("value '%s' is not valid" % value) + + try: + return int(number) * (_TIME_QUALIFIER[qualifier] if qualifier != "" else _TIME_MULTIPLIER_DEFAULT) + except ValueError: + raise ToolError("value '%s' is not valid" % value) diff --git a/build/lib/config/parse.py b/build/lib/config/parse.py new file mode 100644 index 000000000..5bc78c064 --- /dev/null +++ b/build/lib/config/parse.py @@ -0,0 +1,708 @@ +"""Parse Configuration Yaml. + +Reads the command and option declarations, which are the definition of record for the pgBackRest command line and configuration file. + +Most of the work here is inheritance. An option can take another option's command list, allow list, or dependency, and a command entry +in an option can default to what the option itself declares, so what a declaration means depends on what was declared before it. That +is resolved here rather than at render time, so the render sees complete options. + +The declaration order is also what the reader sees, so an option can only inherit from an option above it.""" + +#################################################################################################################################### +import copy +import os + +from common.error import ToolError, check +from common.storage import file_read +from common.yaml import YamlMap, yaml_bool, yaml_load, yaml_map_empty + +# Command roles. A role is a process that carries out part of a command, and main is the process the user started. The order here is +# the order the roles are rendered in. +CMD_ROLE_MAIN = "main" +CMD_ROLE_ASYNC = "async" +CMD_ROLE_LOCAL = "local" +CMD_ROLE_REMOTE = "remote" + +CMD_ROLE_LIST = (CMD_ROLE_MAIN, CMD_ROLE_ASYNC, CMD_ROLE_LOCAL, CMD_ROLE_REMOTE) + +# Commands that take no options of their own, so they are left out of an option's command list +CMD_HELP = "help" +CMD_VERSION = "version" + +# Option types +OPT_TYPE_BOOLEAN = "boolean" +OPT_TYPE_HASH = "hash" +OPT_TYPE_INTEGER = "integer" +OPT_TYPE_LIST = "list" +OPT_TYPE_PATH = "path" +OPT_TYPE_SIZE = "size" +OPT_TYPE_STRING = "string" +OPT_TYPE_STRING_ID = "string-id" +OPT_TYPE_TIME = "time" + +# Options that the rest of the build refers to by name +OPT_STANZA = "stanza" + +# Sections an option can be configured in. An option in the command-line section cannot be set in the configuration file, which is +# why it can be neither negated nor reset. +SECTION_COMMAND_LINE = "command-line" + +# How a default value is rendered +DEFAULT_TYPE_QUOTE = "quote" +DEFAULT_TYPE_LITERAL = "literal" +DEFAULT_TYPE_DYNAMIC = "dynamic" + +_DEFAULT_TYPE_LIST = (DEFAULT_TYPE_QUOTE, DEFAULT_TYPE_LITERAL, DEFAULT_TYPE_DYNAMIC) + + +#################################################################################################################################### +class BldCfgCommand: + """A command.""" + + def __init__(self, name): + self.name = name + self.internal = False # Is the command internal? + self.log_file = True # Does the command write automatically to a log file? + self.log_level_default = "info" # Default log level + self.lock_required = False # Is a lock required? + self.lock_remote_required = False # Is a remote lock required? + self.lock_type = "none" # Lock type + self.parameter_allowed = False # Are command line parameters allowed? + self.role_list = None # Roles valid for the command + + +#################################################################################################################################### +class BldCfgOptionGroup: + """An option group, i.e. a set of options that can be indexed, e.g. repo1-path and repo2-path.""" + + def __init__(self, name): + self.name = name + + +#################################################################################################################################### +class BldCfgOptionValue: + """A value in an allow list, which may be compiled in only when a feature is.""" + + def __init__(self, value, condition): + self.value = value + self.condition = condition # Is the value conditionally compiled? + + +#################################################################################################################################### +class BldCfgOptionAllowRangeMap: + """An allowed range for one value of the option the range maps on.""" + + def __init__(self, map, min, max): + self.map = map + self.min = min + self.max = max + + +#################################################################################################################################### +class BldCfgOptionAllowRange: + """An allowed range, either one range or a range per value of another option.""" + + def __init__(self, min=None, max=None, map_list=None): + self.min = min + self.max = max + self.map_list = map_list + + +#################################################################################################################################### +class BldCfgOptionDefaultMap: + """A default for one value of the option the default maps on.""" + + def __init__(self, map, value): + self.map = map + self.value = value + + +#################################################################################################################################### +class BldCfgOptionDefault: + """A default value, either one value or a value per value of another option.""" + + def __init__(self, value=None, map_list=None): + self.value = value + self.map_list = map_list + + +#################################################################################################################################### +class BldCfgOptionDepend: + """A dependency on another option.""" + + def __init__(self, option, default_value, value_list): + self.option = option # Option the dependency is on + self.default_value = default_value # Value to use when the dependency is not resolved + self.value_list = value_list # Values of the option that resolve the dependency + + +#################################################################################################################################### +class BldCfgOptionDeprecate: + """A deprecated name for an option.""" + + def __init__(self, name): + self.name = name + self.indexed = False # Can the deprecation be indexed? + self.unindexed = False # Can the deprecation be unindexed? + + +#################################################################################################################################### +class BldCfgOptionCommand: + """What an option means for one command.""" + + def __init__(self, name): + self.name = name + self.internal = None # Is the option internal, or None to use what the option declares? + self.required = None # Is the option required, or None to use what the option declares? + self.sequence = None # Is a sequence added to the StringId, or None to use what the option declares? + self.default_value = None + self.depend = None + self.allow_list = None + self.role_list = None # Roles of the command the option is valid for + + +#################################################################################################################################### +class BldCfgOption: + """An option.""" + + def __init__(self, name): + self.name = name + self.type = None # Option type, e.g. integer + self.section = None # Section the option can be configured in + self.bool_like = False # Does the option accept y/n and can be treated as a boolean? + self.internal = False # Is the option internal? + self.beta = False # Is the option beta? + self.required = True # Is the option required? + self.negate = None # Can the option be negated? + self.reset = False # Can the option be reset? + self.sequence = False # Is a sequence added to the StringId? + self.default_type = DEFAULT_TYPE_QUOTE # How the default is rendered + self.default_value = None + self.group = None # Option group, if any + self.secure = False # Does the option contain a secret? + self.depend = None + self.allow_list = None + self.allow_range = None + self.cmd_list = None # What the option means for each command it is valid for + self.cmd_role_list = None # Roles the option is valid for, before they are filtered per command + self.deprecate_list = None + + +#################################################################################################################################### +class BldCfg: + """The configuration declaration.""" + + def __init__(self, cmd_list, opt_grp_list, opt_list, opt_resolve_list): + self.cmd_list = cmd_list # Commands, sorted + self.opt_grp_list = opt_grp_list # Option groups, sorted + self.opt_list = opt_list # Options, sorted + self.opt_resolve_list = opt_resolve_list # Options in the order their dependencies can be resolved in + + +#################################################################################################################################### +def _find(item_list, name): + """Find an entry in a list by name, or None when it is not there.""" + + for item in item_list: + if item.name == name: + return item + + return None + + +#################################################################################################################################### +def _command_role(role_raw, name): + """Parse a command role list, which is a map of roles with nothing under them.""" + + result = [] + + for role, detail in role_raw: + yaml_map_empty(detail, "%s role '%s'" % (name, role)) + result.append(role) + + return result + + +#################################################################################################################################### +def _command_list(cmd_raw): + """Parse the command list.""" + + result = [] + + for name, detail in cmd_raw: + cmd = BldCfgCommand(name) + + for key, value in detail: + if key == "command-role": + cmd.role_list = _command_role(value, "command '%s'" % name) + elif key == "internal": + cmd.internal = yaml_bool(value, "command '%s' internal" % name) + elif key == "lock-type": + cmd.lock_type = value + elif key == "lock-remote-required": + cmd.lock_remote_required = yaml_bool(value, "command '%s' lock-remote-required" % name) + elif key == "lock-required": + cmd.lock_required = yaml_bool(value, "command '%s' lock-required" % name) + elif key == "log-file": + cmd.log_file = yaml_bool(value, "command '%s' log-file" % name) + elif key == "log-level-default": + cmd.log_level_default = value.lower() + elif key == "parameter-allowed": + cmd.parameter_allowed = yaml_bool(value, "command '%s' parameter-allowed" % name) + else: + raise ToolError("unknown command definition '%s'" % key) + + # Every command has a main role whether it declares one or not, since main is the process the user started + cmd.role_list = sorted(set(cmd.role_list or []) | {CMD_ROLE_MAIN}) + + result.append(cmd) + + return sorted(result, key=lambda cmd: cmd.name) + + +#################################################################################################################################### +def _option_group_list(opt_grp_raw): + """Parse the option group list, which is a map of groups with nothing under them.""" + + result = [] + + for name, detail in opt_grp_raw: + yaml_map_empty(detail, "option group '%s'" % name) + result.append(BldCfgOptionGroup(name)) + + return sorted(result, key=lambda opt_grp: opt_grp.name) + + +#################################################################################################################################### +def _allow_list(allow_raw, opt_list, name): + """Parse an allow list, which is either the list of allowed values or the name of an option to take it from.""" + + # A scalar names the option the allow list is inherited from + if isinstance(allow_raw, str): + check(opt_list is not None, "allow list for %s cannot be inherited" % name) + + opt_inherit = _find(opt_list, allow_raw) + check(opt_inherit is not None, "allow list inherited from option '%s' before it is defined" % allow_raw) + + return copy.copy(opt_inherit.allow_list) + + result = [] + + for value in allow_raw: + # A scalar is the value on its own, else a map of the value to the feature it is compiled in with + if isinstance(value, str): + result.append(BldCfgOptionValue(value, None)) + else: + check(len(value) == 1, "allow list value for %s must have a single condition" % name) + + for item, condition in value: + result.append(BldCfgOptionValue(item, condition)) + + return result + + +#################################################################################################################################### +def _allow_range(range_raw, name): + """Parse an allow range, which is either a min/max pair or a min/max pair per value of the option it maps on.""" + + # A scalar first means the range itself rather than a range per map + if isinstance(range_raw[0], str): + check(len(range_raw) == 2, "allow range for %s must be a min and a max" % name) + + return BldCfgOptionAllowRange(min=range_raw[0], max=range_raw[1]) + + map_list = [] + + for entry in range_raw: + for map, range in entry: + check(len(range) == 2, "allow range for %s map '%s' must be a min and a max" % (name, map)) + map_list.append(BldCfgOptionAllowRangeMap(map, range[0], range[1])) + + return BldCfgOptionAllowRange(map_list=map_list) + + +#################################################################################################################################### +def _default(default_raw): + """Parse a default, which is either the value or a value per value of the option it maps on. + + A tilde overrides an inherited default back to no default at all.""" + + # A scalar is the default itself + if isinstance(default_raw, str): + return None if default_raw == "~" else BldCfgOptionDefault(value=default_raw) + + map_list = [] + + for entry in default_raw: + for map, value in entry: + map_list.append(BldCfgOptionDefaultMap(map, value)) + + return BldCfgOptionDefault(map_list=map_list) + + +#################################################################################################################################### +def _depend(depend_raw, opt_list): + """Parse a dependency, which is either the dependency or the name of an option to take it from.""" + + # A scalar names the option the dependency is inherited from + if isinstance(depend_raw, str): + opt_inherit = _find(opt_list, depend_raw) + + if opt_inherit is None: + raise ToolError("dependency inherited from option '%s' before it is defined" % depend_raw) + + return opt_inherit.depend + + option = None + default_value = None + value_list = None + + for key, value in depend_raw: + if key == "list": + value_list = list(value) + elif key == "default": + default_value = value + elif key == "option": + option = value + else: + raise ToolError("unknown depend definition '%s'" % key) + + return BldCfgOptionDepend(option, default_value, value_list) + + +#################################################################################################################################### +def _depend_reconcile(opt, depend, opt_list): + """Resolve a dependency to the option it is on, now that every option exists.""" + + if depend is None: + return None + + if depend.default_value is not None and opt.type not in (OPT_TYPE_BOOLEAN, OPT_TYPE_INTEGER): + raise ToolError("dependency default invalid for non integer/boolean option '%s'" % opt.name) + + opt_depend = _find(opt_list, depend.option) + + if opt_depend is None: + raise ToolError("dependency on undefined option '%s'" % depend.option) + + return BldCfgOptionDepend(opt_depend, depend.default_value, depend.value_list) + + +#################################################################################################################################### +def _deprecate_list(deprecate_raw): + """Parse the deprecated names of an option. + + A question mark in the name means the deprecation may be indexed, e.g. repo?-path covers repo-path and repo1-path, so the same + name can appear twice and each spelling is recorded separately.""" + + result = [] + + for name, detail in deprecate_raw: + yaml_map_empty(detail, "deprecate '%s'" % name) + + indexed = "?" in name + name = name.replace("?", "", 1) + + deprecate = _find(result, name) + + if deprecate is None: + deprecate = BldCfgOptionDeprecate(name) + result.append(deprecate) + + if indexed: + deprecate.indexed = True + else: + deprecate.unindexed = True + + return sorted(result, key=lambda deprecate: deprecate.name) + + +#################################################################################################################################### +def _option_command_list(opt_cmd_raw, cmd_list, opt_list, name): + """Parse the commands an option is valid for, which is either the list or the name of an option to take it from.""" + + # A scalar names the option the command list is inherited from + if isinstance(opt_cmd_raw, str): + opt_inherit = _find(opt_list, opt_cmd_raw) + check(opt_inherit is not None, "command list inherited from option '%s' before it is defined" % opt_cmd_raw) + + return opt_inherit.cmd_list + + result = [] + + for opt_cmd, detail in opt_cmd_raw: + # Add every command of another option, stripped of what that option declared about them + if opt_cmd == "+inherit": + opt_inherit = _find(opt_list, detail) + check(opt_inherit is not None, "commands inherited from option '%s' before it is defined" % detail) + + for cmd in opt_inherit.cmd_list: + result.append(BldCfgOptionCommand(cmd.name)) + # Add every command that has a role, or any role + elif opt_cmd == "+role": + for cmd in cmd_list: + # Help and version take no options of their own + if cmd.name in (CMD_HELP, CMD_VERSION): + continue + + for role in cmd.role_list: + if (detail == "any" or role == detail) and _find(result, cmd.name) is None: + result.append(BldCfgOptionCommand(cmd.name)) + # Remove a command, which is how a command is trimmed from what +inherit or +role added + elif opt_cmd == "-command": + cmd = _find(result, detail) + + if cmd is not None: + result.remove(cmd) + # Else the command is declared with what it overrides + else: + cmd = BldCfgOptionCommand(opt_cmd) + + for key, value in detail: + if key == "allow-list": + cmd.allow_list = _allow_list(value, None, "%s command '%s'" % (name, opt_cmd)) + elif key == "command-role": + cmd.role_list = _command_role(value, "%s command '%s'" % (name, opt_cmd)) + elif key == "depend": + cmd.depend = _depend(value, opt_list) + elif key == "default": + cmd.default_value = _default(value) + elif key == "internal": + cmd.internal = yaml_bool(value, "%s command '%s' internal" % (name, opt_cmd)) + elif key == "required": + cmd.required = yaml_bool(value, "%s command '%s' required" % (name, opt_cmd)) + elif key == "sequence": + cmd.sequence = yaml_bool(value, "%s command '%s' sequence" % (name, opt_cmd)) + else: + raise ToolError("unknown option command definition '%s'" % key) + + # A declaration replaces whatever +inherit or +role added for the command + cmd_prior = _find(result, opt_cmd) + + if cmd_prior is not None: + result.remove(cmd_prior) + + result.append(cmd) + + return sorted(result, key=lambda cmd: cmd.name) + + +#################################################################################################################################### +def _option_list_raw(opt_raw, cmd_list, opt_grp_list): + """Parse the options as declared, before any of them are resolved against each other.""" + + result = [] + + for name, detail in opt_raw: + opt = BldCfgOption(name) + inherit_found = False + + for key, value in detail: + if key == "allow-list": + opt.allow_list = _allow_list(value, result, "option '%s'" % name) + elif key == "allow-range": + opt.allow_range = _allow_range(value, "option '%s'" % name) + elif key == "command": + opt.cmd_list = _option_command_list(value, cmd_list, result, "option '%s'" % name) + elif key == "command-role": + opt.cmd_role_list = _command_role(value, "option '%s'" % name) + elif key == "default": + opt.default_value = _default(value) + elif key == "depend": + opt.depend = _depend(value, result) + elif key == "deprecate": + opt.deprecate_list = _deprecate_list(value) + elif key == "default-type": + check(value in _DEFAULT_TYPE_LIST, "option '%s' has invalid default type '%s'" % (name, value)) + opt.default_type = value + elif key == "group": + check(_find(opt_grp_list, value) is not None, "option '%s' has invalid group '%s'" % (name, value)) + opt.group = value + elif key == "inherit": + opt_inherit = _find(result, value) + check(opt_inherit is not None, "option '%s' inherited before it is defined" % value) + + # Everything declared before inherit is replaced and everything after it overrides, which is why inherit is written + # first. Deprecations are not inherited since they name the option they are a deprecation of. + opt = copy.copy(opt_inherit) + opt.name = name + opt.deprecate_list = None + + inherit_found = True + elif key == "internal": + opt.internal = yaml_bool(value, "option '%s' internal" % name) + elif key == "bool-like": + opt.bool_like = yaml_bool(value, "option '%s' bool-like" % name) + elif key == "beta": + opt.beta = yaml_bool(value, "option '%s' beta" % name) + elif key == "negate": + opt.negate = yaml_bool(value, "option '%s' negate" % name) + elif key == "sequence": + opt.sequence = yaml_bool(value, "option '%s' sequence" % name) + elif key == "required": + opt.required = yaml_bool(value, "option '%s' required" % name) + elif key == "section": + opt.section = value + elif key == "secure": + opt.secure = yaml_bool(value, "option '%s' secure" % name) + elif key == "type": + opt.type = value + else: + raise ToolError("unknown option definition '%s'" % key) + + if opt.type is None: + raise ToolError("option '%s' requires 'type'" % name) + + # An inherited option already has these set from the option it inherited from + if not inherit_found: + # An option that does not say where it can be configured can only be given on the command line + if opt.section is None: + opt.section = SECTION_COMMAND_LINE + + # Only an option that can be written in the configuration file can be negated there + if opt.negate is None: + opt.negate = (opt.type == OPT_TYPE_BOOLEAN or opt.bool_like) and opt.section != SECTION_COMMAND_LINE + + # An option that does not name its commands is valid for all of them + if opt.cmd_list is None: + opt.cmd_list = sorted( + (BldCfgOptionCommand(cmd.name) for cmd in cmd_list if cmd.name not in (CMD_HELP, CMD_VERSION)), + key=lambda cmd: cmd.name, + ) + + # Only an option that can be written in the configuration file can be reset, which is what removes it again + opt.reset = opt.section != SECTION_COMMAND_LINE + + result.append(opt) + + return sorted(result, key=lambda opt: opt.name) + + +#################################################################################################################################### +def _option_list(opt_raw, cmd_list, opt_grp_list): + """Parse the option list and resolve every option against the others.""" + + opt_list_raw = _option_list_raw(opt_raw, cmd_list, opt_grp_list) + + # Copy the options so a dependency can point at the option it is on rather than at the name of one + result = [] + + for opt_raw in opt_list_raw: + opt = copy.copy(opt_raw) + opt.cmd_list = None + opt.depend = None + result.append(opt) + + # Resolve what each option means for each of its commands, which is what it declares for the command falling back to what it + # declares for itself + for idx, opt_raw in enumerate(opt_list_raw): + cmd_opt_list = [] + + for opt_cmd_raw in opt_raw.cmd_list: + cmd = _find(cmd_list, opt_cmd_raw.name) + + if cmd is None: + raise ToolError("invalid command '%s' in option '%s' command list" % (opt_cmd_raw.name, opt_raw.name)) + + opt_cmd = copy.copy(opt_cmd_raw) + + if opt_cmd.required is None: + opt_cmd.required = opt_raw.required + + if opt_cmd.internal is None: + opt_cmd.internal = opt_raw.internal + + if opt_cmd.sequence is None: + opt_cmd.sequence = opt_raw.sequence + + if opt_cmd.role_list is None: + # What the option declares for every command, less the roles this command does not have + if opt_raw.cmd_role_list is not None: + opt_cmd.role_list = [role for role in opt_raw.cmd_role_list if role in cmd.role_list] + else: + opt_cmd.role_list = cmd.role_list + + opt_cmd.depend = _depend_reconcile(opt_raw, opt_cmd_raw.depend, result) + cmd_opt_list.append(opt_cmd) + + result[idx].cmd_list = cmd_opt_list + result[idx].depend = _depend_reconcile(opt_raw, opt_raw.depend, result) + + return result + + +#################################################################################################################################### +def _option_resolve_list(opt_list): + """Order the options so that every option comes after the options it depends on. + + The stanza option is resolved first since an error about a missing option is confusing when the stanza it would apply to is what is + actually missing, so stanza must exist and may not depend on anything.""" + + opt_stanza = _find(opt_list, OPT_STANZA) + + if opt_stanza is None: + raise ToolError("option '%s' must exist" % OPT_STANZA) + + if opt_stanza.depend is not None: + raise ToolError("option '%s' may not depend on other option" % OPT_STANZA) + + for opt_cmd in opt_stanza.cmd_list: + if opt_cmd.depend is not None: + raise ToolError("option '%s' command '%s' may not depend on other option" % (OPT_STANZA, opt_cmd.name)) + + resolve_list = [OPT_STANZA] + + while len(resolve_list) != len(opt_list): + resolved = False + + for opt in opt_list: + if opt.name in resolve_list: + continue + + if opt.depend is not None and opt.depend.option.name not in resolve_list: + continue + + if any(cmd.depend is not None and cmd.depend.option.name not in resolve_list for cmd in opt.cmd_list): + continue + + resolve_list.append(opt.name) + resolved = True + + # Nothing resolved means the options left depend on each other, since every other reason to skip is already excluded + if not resolved: + raise ToolError( + "unable to resolve dependencies for option(s) '%s'\nHINT: are there circular dependencies?" + % ", ".join(opt.name for opt in opt_list if opt.name not in resolve_list) + ) + + return [_find(opt_list, name) for name in resolve_list] + + +#################################################################################################################################### +def bld_cfg_parse(path_repo): + """Parse config.yaml into the configuration declaration.""" + + path = os.path.join(path_repo, "build/config.yaml") + cfg_raw = yaml_load(file_read(path), path) + + cmd_raw = None + opt_grp_raw = None + opt_raw = None + + for key, value in cfg_raw: + if key == "command": + cmd_raw = value + elif key == "optionGroup": + opt_grp_raw = value + elif key == "option": + opt_raw = value + else: + raise ToolError("unknown config definition '%s'" % key) + + check(cmd_raw is not None and opt_grp_raw is not None and opt_raw is not None, "command, optionGroup, and option are required") + + cmd_list = _command_list(cmd_raw) + opt_grp_list = _option_group_list(opt_grp_raw) + opt_list = _option_list(opt_raw, cmd_list, opt_grp_list) + + return BldCfg(cmd_list, opt_grp_list, opt_list, _option_resolve_list(opt_list)) diff --git a/build/lib/config/render.py b/build/lib/config/render.py new file mode 100644 index 000000000..59c7faeb6 --- /dev/null +++ b/build/lib/config/render.py @@ -0,0 +1,773 @@ +"""Render Configuration Data. + +Writes the constants and enums the code refers to options and commands by, and the parse rules the command line and configuration file +are read with. + +The parse rules are packed rather than written as plain structures: every value the rules refer to is collected into one array per type +and the rules index into it, so a value used by many options is stored once. That is what most of the work here is. Each rule is also +labelled with the option or command it came from, in a comment at the right margin, so the generated file can be read. + +Rules that come out identical for several commands are emitted once with a filter naming those commands, which is why the rules are +built as text first and grouped by what they turned out to be.""" + +#################################################################################################################################### +import os + +from common.error import ToolError, check +from common.render import COMMENT_SEPARATOR, LINE_LENGTH, bld_comment_block, bld_define, bld_enum, bld_header, bld_str_id_seq +from common.storage import file_write_differs +from common.string_id import STRING_ID_SEQ_NONE +from config.common import cfg_parse_size, cfg_parse_time +from config.parse import ( + CMD_ROLE_LIST, + DEFAULT_TYPE_DYNAMIC, + DEFAULT_TYPE_LITERAL, + OPT_TYPE_BOOLEAN, + OPT_TYPE_HASH, + OPT_TYPE_INTEGER, + OPT_TYPE_LIST, + OPT_TYPE_PATH, + OPT_TYPE_SIZE, + OPT_TYPE_STRING, + OPT_TYPE_STRING_ID, + OPT_TYPE_TIME, +) + +_MODULE = "config" +_CONFIG_DESCRIPTION = "Command and Option Configuration" +_PARSE_DESCRIPTION = "Config Parse Rules" + +# Characters that cannot appear in a C name, and what they are spelled as when a value becomes one +_ENUM_CHAR = {'"': "QT", "/": "FS", " ": "SP", ".": "DT", "-": "DS"} + +# Optional rules, in the order they are rendered in. A rule declared for a command replaces the one declared for the option, so both +# are collected under the same key and the command wins. +_RULE_DEPEND = "01-depend" +_RULE_SEQUENCE = "02-sequence" +_RULE_ALLOW_RANGE = "03-allow-range" +_RULE_ALLOW_LIST = "04-allow-list" +_RULE_DEFAULT = "05-default" +_RULE_REQUIRE = "06-require" + +_RULE_LIST = (_RULE_DEPEND, _RULE_SEQUENCE, _RULE_ALLOW_RANGE, _RULE_ALLOW_LIST, _RULE_DEFAULT, _RULE_REQUIRE) + + +#################################################################################################################################### +def _const(prefix, value): + """Build a constant name from a value, e.g. "repo-path" becomes "CFGOPT_REPO_PATH".""" + + return ("%s_%s" % (prefix, value)).replace("-", "_").upper() + + +#################################################################################################################################### +def _enum_str(source): + """Build the part of a C name that stands for a value. + + A value can hold characters a name cannot, so each is spelled out and separated from what surrounds it, e.g. "9.6" becomes + "9_DT_6" and "\"none\"" becomes "QT_none_QT".""" + + result = "" + prior_special = False + + for idx, char in enumerate(source): + if char not in _ENUM_CHAR: + result += char + prior_special = False + + continue + + if idx != 0 and not prior_special: + result += "_" + + result += _ENUM_CHAR[char] + + if idx != len(source) - 1: + result += "_" + + prior_special = True + + return result + + +#################################################################################################################################### +def _var_128_size(value): + """Bytes a value takes when it is encoded as a variable length integer, which is how a rule stores an index.""" + + result = 1 + + while value >= 0x80: + value >>= 7 + result += 1 + + return result + + +#################################################################################################################################### +def _label(content, label, label_text): + """Label every line of a rule with what it came from, in a comment at the right margin. + + A line that is already too long to fit the comment is left alone, since wrapping it would be worse than leaving it unlabelled. + """ + + if not label: + return content + + comment = "// %s" % label_text + result = [] + + for line in content.split("\n"): + result.append(line if len(line) + 1 + len(comment) > LINE_LENGTH else line + comment.rjust(LINE_LENGTH - len(line))) + + return "\n".join(result) + + +#################################################################################################################################### +def _scalar(scalar, opt_type): + """Render a value as the rule that refers to it, which is an index into the values of its type.""" + + if opt_type == OPT_TYPE_STRING_ID: + return "PARSE_RULE_VAL_STRID(%s)" % _enum_str(scalar) + + if opt_type == OPT_TYPE_STRING: + return "PARSE_RULE_VAL_STR(%s)" % _enum_str(scalar) + + if opt_type == OPT_TYPE_BOOLEAN: + return "PARSE_RULE_VAL_BOOL_%s" % ("TRUE" if scalar == "true" else "FALSE") + + if opt_type == OPT_TYPE_INTEGER: + return "PARSE_RULE_VAL_INT(%s)" % _enum_str(scalar) + + if opt_type == OPT_TYPE_SIZE: + return "PARSE_RULE_VAL_SIZE(%s)" % scalar + + return "PARSE_RULE_VAL_TIME(%s)" % scalar + + +#################################################################################################################################### +def _render_depend(opt_type, depend): + """Render what an option depends on, i.e. the option and the values of it that make this option valid.""" + + result = " PARSE_RULE_OPTIONAL_DEPEND\n (\n" + + if depend.default_value is not None: + result += " PARSE_RULE_OPTIONAL_DEPEND_DEFAULT(%s),\n" % _scalar(depend.default_value, opt_type) + + result += " PARSE_RULE_VAL_OPT(%s),\n" % bld_enum("", depend.option.name) + + if depend.value_list is not None: + # A string option is compared as a StringId here, since a dependency is on one of a small set of known values + type_depend = OPT_TYPE_STRING_ID if depend.option.type == OPT_TYPE_STRING else depend.option.type + + for value in depend.value_list: + result += " %s,\n" % _scalar(value, type_depend) + + return result + " )" + + +#################################################################################################################################### +def _render_allow_range(allow_range, opt_type): + """Render the range a value must be in, which may be a range per value of the option the range maps on.""" + + result = " PARSE_RULE_OPTIONAL_ALLOW_RANGE\n (\n" + + if allow_range.map_list is not None: + result += " PARSE_RULE_OPTIONAL_ALLOW_RANGE_MAP\n (\n" + + for map in allow_range.map_list: + result += " %s,\n" % _scalar(map.map, OPT_TYPE_STRING_ID) + result += " %s,\n" % _scalar(map.min, opt_type) + result += " %s,\n" % _scalar(map.max, opt_type) + + result += " ),\n" + else: + result += " %s,\n" % _scalar(allow_range.min, opt_type) + result += " %s,\n" % _scalar(allow_range.max, opt_type) + + return result + " )" + + +#################################################################################################################################### +def _render_allow_list(allow_list, opt_type): + """Render the values an option allows. + + A value that is only compiled in with a feature is replaced by false when the feature is not, so the value keeps its index and the + rules that refer to it by index stay correct.""" + + result = " PARSE_RULE_OPTIONAL_ALLOW_LIST\n (\n" + + for allow in allow_list: + value = '"%s"' % allow.value if opt_type == OPT_TYPE_STRING else allow.value + + result += " %s,\n" % _scalar(value, opt_type) + + if allow.condition is not None: + result += "#ifndef %s\n PARSE_RULE_BOOL_FALSE,\n#endif\n" % allow.condition + + return result + " )" + + +#################################################################################################################################### +def _render_default_value(value, default_type, opt_type, indent): + """Render one default value.""" + + result = " " if indent else "" + + # A string or path default is rendered as a string, quoted unless it is written as the literal C to use + if opt_type in (OPT_TYPE_STRING, OPT_TYPE_PATH): + quote = "" if default_type == DEFAULT_TYPE_LITERAL else '"' + value = _scalar("%s%s%s" % (quote, value, quote), OPT_TYPE_STRING) + else: + value = _scalar(value, opt_type) + + return result + " %s,\n" % value + + +#################################################################################################################################### +def _render_default(default, default_type, opt_type, sequence, allow_list): + """Render the default of an option, which may be a value per value of the option the default maps on.""" + + result = " PARSE_RULE_OPTIONAL_DEFAULT\n (\n" + + # A dynamic default is worked out at run time, so only the name of what works it out is rendered + if default_type == DEFAULT_TYPE_DYNAMIC: + result += " PARSE_RULE_DEFAULT_DYNAMIC(%s),\n" % bld_enum("", default.value) + elif default.value is not None: + result += _render_default_value(default.value, default_type, opt_type, False) + + # An option with a sequence also needs where the default sits in the allow list, since that is what the sequence counts + if sequence: + allow_idx = next((idx for idx, allow in enumerate(allow_list) if allow.value == default.value), None) + + if allow_idx is None: + raise ToolError("unable to find default '%s' in allow list" % default.value) + + if allow_idx != 0: + result += " PARSE_RULE_VAL_SEQ(%u),\n" % allow_idx + else: + result += " PARSE_RULE_OPTIONAL_DEFAULT_MAP\n (\n" + + for map in default.map_list: + result += " %s,\n" % _scalar(map.map, OPT_TYPE_STRING_ID) + result += _render_default_value(map.value, default_type, opt_type, True) + + result += " ),\n" + + return result + " )" + + +#################################################################################################################################### +def _value_add(opt_type, default_type, value, rule_val_map): + """Record a value so it gets an entry in the values of its type, which is what the rules index into.""" + + # A dynamic default is worked out at run time so there is no value to store + if default_type == DEFAULT_TYPE_DYNAMIC: + return + + # A path is stored as a string + if opt_type == OPT_TYPE_PATH: + opt_type = OPT_TYPE_STRING + + quote = "" if default_type == DEFAULT_TYPE_LITERAL else '"' + value_str = "%s%s%s" % (quote, value, quote) + + # A value of any other type also has a string, since a value is reported as the text it was written as + if opt_type != OPT_TYPE_STRING: + rule_val_map.setdefault(opt_type, {})[value] = value_str + + rule_val_map.setdefault(OPT_TYPE_STRING, {})[value_str] = None + + +#################################################################################################################################### +def _value_render(opt_type, rule_val_map, label, type, caption, abbr, macro, comment): + """Render the values of one type, the map from each to its string, and the enum the rules index them by.""" + + # Every type needs a value, since the array the rules index into cannot be empty + check(opt_type in rule_val_map, "declaration has no %s value for the rules to index" % opt_type) + + rule_val_list = list(rule_val_map[opt_type]) + + # Sort by what the value means rather than by how it is written, so a range check on the index is a range check on the value + if opt_type == OPT_TYPE_INTEGER: + rule_val_list.sort(key=lambda value: int(value)) + elif opt_type == OPT_TYPE_SIZE: + rule_val_list.sort(key=cfg_parse_size) + elif opt_type == OPT_TYPE_TIME: + rule_val_list.sort(key=cfg_parse_time) + else: + rule_val_list.sort() + + result = "\n" + bld_comment_block("Rule %ss" % caption) + result += ( + bld_define( + "PARSE_RULE_VAL_%s(value)" % macro, + "PARSE_RULE_U32_%u(parseRuleVal%s##value)" % (_var_128_size(len(rule_val_list) - 1), abbr), + ) + + "\n" + ) + + # A StringId is stored as the string it spells, so it refers to the string values + if opt_type == OPT_TYPE_STRING: + result += bld_define("PARSE_RULE_VAL_STRID(value)", "PARSE_RULE_VAL_STR(QT_##value##_QT)") + "\n" + + result += "\nstatic const %s parseRuleValue%s[] =\n{\n" % (type, abbr) + + value_list = [] + + for value in rule_val_list: + if opt_type == OPT_TYPE_STRING: + value_list.append(" PARSE_RULE_STRPUB(%s)," % value) + elif opt_type == OPT_TYPE_INTEGER: + value_list.append(" %s," % value.strip()) + elif opt_type == OPT_TYPE_SIZE: + value_list.append(" %d," % cfg_parse_size(value)) + else: + value_list.append(" %d," % cfg_parse_time(value)) + + result += _label("\n".join(value_list), label, comment) + "\n};\n" + + # Map each value to its string, so a value can be reported as the text it was written as + if opt_type in (OPT_TYPE_INTEGER, OPT_TYPE_SIZE, OPT_TYPE_TIME): + result += "\nstatic const uint8_t parseRuleValue%sStrMap[] =\n{\n" % abbr + + map_list = [" parseRuleValStr%s," % _enum_str(rule_val_map[opt_type][value]) for value in rule_val_list] + + result += _label("\n".join(map_list), label, "%s/strmap" % comment) + "\n};\n" + + result += "\ntypedef enum\n{\n" + + enum_list = [] + + for value in rule_val_list: + name = _enum_str(value) if opt_type in (OPT_TYPE_STRING, OPT_TYPE_INTEGER) else bld_enum("", value) + enum_list.append(" parseRuleVal%s%s," % (abbr, name)) + + result += _label("\n".join(enum_list), label, "%s/enum" % comment) + + return result + "\n} ParseRuleValue%s;\n" % abbr + + +#################################################################################################################################### +def _render_config_auto_h(bld_cfg): + """Render config.auto.h, which is what the code refers to commands, options, and option values by.""" + + result = bld_header(_MODULE, _CONFIG_DESCRIPTION) + "#ifndef CONFIG_CONFIG_AUTO_H\n#define CONFIG_CONFIG_AUTO_H\n" + + # Command constants + result += "\n" + bld_comment_block("Command constants") + + for cmd in bld_cfg.cmd_list: + result += bld_define(_const("CFGCMD", cmd.name), '"%s"' % cmd.name) + "\n" + + result += "\n" + bld_define("CFG_COMMAND_TOTAL", "%u" % len(bld_cfg.cmd_list)) + "\n" + + # Option group constants + result += "\n" + bld_comment_block("Option group constants") + result += bld_define("CFG_OPTION_GROUP_TOTAL", "%u" % len(bld_cfg.opt_grp_list)) + "\n" + + # Option constants. An option in a group is referred to by the group and an index rather than by name. + result += "\n" + bld_comment_block("Option constants") + + for opt in bld_cfg.opt_list: + if opt.group is None: + result += bld_define(_const("CFGOPT", opt.name), '"%s"' % opt.name) + "\n" + + result += "\n" + bld_define("CFG_OPTION_TOTAL", "%u" % len(bld_cfg.opt_list)) + "\n" + + # Option value constants + result += "\n" + bld_comment_block("Option value constants") + + lf = False + + for opt in bld_cfg.opt_list: + if opt.type != OPT_TYPE_STRING_ID: + continue + + # Values the option allows for every command, followed by the values it allows for one command only + value_list = [] + group_list = [("", value_list)] + + for allow in opt.allow_list or []: + if allow.value not in value_list: + value_list.append(allow.value) + + for opt_cmd in opt.cmd_list: + if opt_cmd.allow_list is None: + continue + + cmd_value_list = [] + + for allow in opt_cmd.allow_list: + # A sequence numbers the values, so the command keeps its own list rather than adding to the option's + if opt_cmd.sequence: + cmd_value_list.append(allow.value) + elif allow.value not in value_list: + value_list.append(allow.value) + + group_list.append(("%s_" % opt_cmd.name.upper(), cmd_value_list)) + + if not opt.sequence: + value_list.sort() + + for group_idx, (group, allow_list) in enumerate(group_list): + sequence = (group_idx == 0 and opt.sequence) or (group_idx != 0 and len(allow_list) > 0) + + if lf and len(allow_list) > 0: + result += "\n" + + for allow_idx, allow in enumerate(allow_list): + const = _const("CFGOPTVAL", "%s%s_%s" % (group, opt.name, allow)) + + if sequence: + result += bld_define(const, "%u" % allow_idx) + "\n" + + result += ( + bld_define( + "%s%s" % (const, "_STRID" if sequence else ""), + bld_str_id_seq(allow, allow_idx if sequence else STRING_ID_SEQ_NONE), + ) + + "\n" + ) + + result += bld_define("%s_Z" % const, '"%s"' % allow) + "\n" + + lf = True + + # Enums + result += "\n" + bld_comment_block("Command enum") + "typedef enum\n{\n" + result += "".join(" %s,\n" % bld_enum("cfgCmd", cmd.name) for cmd in bld_cfg.cmd_list) + result += "} ConfigCommand;\n" + + result += "\n" + bld_comment_block("Option group enum") + "typedef enum\n{\n" + result += "".join(" %s,\n" % bld_enum("cfgOptGrp", opt_grp.name) for opt_grp in bld_cfg.opt_grp_list) + result += "} ConfigOptionGroup;\n" + + result += "\n" + bld_comment_block("Option enum") + "typedef enum\n{\n" + result += "".join(" %s,\n" % bld_enum("cfgOpt", opt.name) for opt in bld_cfg.opt_list) + result += "} ConfigOption;\n" + + return result + "\n#endif\n" + + +#################################################################################################################################### +def _render_option(opt, bld_cfg, rule_val_map, dynamic_default_list): + """Render the parse rules for one option.""" + + if opt.default_type == DEFAULT_TYPE_DYNAMIC and opt.default_value.value not in dynamic_default_list: + dynamic_default_list.append(opt.default_value.value) + + result = ' PARSE_RULE_OPTION\n (\n PARSE_RULE_OPTION_NAME("%s"),\n' % opt.name + result += " PARSE_RULE_OPTION_TYPE(%s),\n" % bld_enum("", opt.type) + + if opt.sequence: + result += " PARSE_RULE_OPTION_SEQUENCE(true),\n" + + # An internal option is not documented and is not shown in the help + if opt.internal: + result += " PARSE_RULE_OPTION_INTERNAL(true)\n" + + if opt.default_type == DEFAULT_TYPE_DYNAMIC: + result += " PARSE_RULE_OPTION_DEFAULT_TYPE(Dynamic),\n" + + if opt.bool_like: + result += " PARSE_RULE_OPTION_BOOL_LIKE(true),\n" + + if opt.beta: + result += " PARSE_RULE_OPTION_BETA(true),\n" + + if opt.negate: + result += " PARSE_RULE_OPTION_NEGATE(true),\n" + + if opt.reset: + result += " PARSE_RULE_OPTION_RESET(true),\n" + + result += " PARSE_RULE_OPTION_REQUIRED(%s),\n" % ("true" if opt.required else "false") + result += " PARSE_RULE_OPTION_SECTION(%s),\n" % bld_enum("", opt.section) + + if opt.secure: + result += " PARSE_RULE_OPTION_SECURE(true),\n" + + # A hash or a list can be given more than once, which builds up the value rather than replacing it + if opt.type in (OPT_TYPE_HASH, OPT_TYPE_LIST): + result += " PARSE_RULE_OPTION_MULTI(true),\n" + + if opt.group is not None: + result += " PARSE_RULE_OPTION_GROUP_ID(%s),\n" % bld_enum("", opt.group) + + # An option in a group can be given without an index when a deprecation says so, e.g. repo-path for repo1-path + for deprecate in opt.deprecate_list or []: + if deprecate.name == opt.name and deprecate.unindexed: + result += " PARSE_RULE_OPTION_DEPRECATE_MATCH(true),\n" + break + + # Commands the option is internal for, when that differs from the option itself + cmd_internal = "".join( + " PARSE_RULE_OPTION_COMMAND_INTERNAL(%s, %s),\n" % (bld_enum("", cmd.name), "true" if cmd.internal else "false") + for cmd in opt.cmd_list + if cmd.internal != opt.internal + ) + + if cmd_internal != "": + result += "\n PARSE_RULE_OPTION_COMMAND_INTERNAL_LIST\n (\n%s )\n" % cmd_internal + + # Commands the option is valid for, per role, since a role only gets the options its part of the command needs + for role in CMD_ROLE_LIST: + cmd_role = "".join( + " PARSE_RULE_OPTION_COMMAND(%s)\n" % bld_enum("", cmd.name) for cmd in opt.cmd_list if role in cmd.role_list + ) + + if cmd_role != "": + result += "\n PARSE_RULE_OPTION_COMMAND_ROLE_%s_VALID_LIST\n (\n%s ),\n" % (role.upper(), cmd_role) + + # Rules that apply to the option whatever the command + rule_default = {} + + if opt.depend is not None: + rule_default[_RULE_DEPEND] = _render_depend(opt.type, opt.depend) + + if opt.allow_range is not None: + rule_default[_RULE_ALLOW_RANGE] = _render_allow_range(opt.allow_range, opt.type) + + if opt.allow_range.map_list is not None: + for map in opt.allow_range.map_list: + _value_add(opt.type, False, map.min, rule_val_map) + _value_add(opt.type, False, map.max, rule_val_map) + else: + _value_add(opt.type, False, opt.allow_range.min, rule_val_map) + _value_add(opt.type, False, opt.allow_range.max, rule_val_map) + + if opt.allow_list is not None: + rule_default[_RULE_ALLOW_LIST] = _render_allow_list(opt.allow_list, opt.type) + + for allow in opt.allow_list: + _value_add(opt.type, False, allow.value, rule_val_map) + + if opt.default_value is not None: + rule_default[_RULE_DEFAULT] = _render_default(opt.default_value, opt.default_type, opt.type, opt.sequence, opt.allow_list) + + # A boolean default is rendered as the boolean itself rather than as an index into a value list + if opt.type != OPT_TYPE_BOOLEAN: + if opt.default_value.value is not None: + _value_add(opt.type, opt.default_type, opt.default_value.value, rule_val_map) + + for map in opt.default_value.map_list or []: + _value_add(opt.type, opt.default_type, map.value, rule_val_map) + + # Rules that apply only to some commands, which are the rules declared for the command falling back to the rules above + rule_cmd = {} + + for opt_cmd in opt.cmd_list: + rule_cmd_type = {} + + if opt_cmd.depend is not None: + rule_cmd_type[_RULE_DEPEND] = _render_depend(opt.type, opt_cmd.depend) + + if opt_cmd.sequence != opt.sequence: + rule_cmd_type[_RULE_SEQUENCE] = " PARSE_RULE_OPTIONAL_%sSEQUENCE()" % ( + "" if opt_cmd.required else "NOT_" + ) + + if opt_cmd.allow_list is not None: + rule_cmd_type[_RULE_ALLOW_LIST] = _render_allow_list(opt_cmd.allow_list, opt.type) + + for allow in opt_cmd.allow_list: + _value_add(opt.type, False, allow.value, rule_val_map) + + if opt_cmd.default_value is not None: + rule_cmd_type[_RULE_DEFAULT] = _render_default( + opt_cmd.default_value, opt.default_type, opt.type, opt.sequence, opt_cmd.allow_list + ) + + if opt.type != OPT_TYPE_BOOLEAN: + _value_add(opt.type, opt.default_type, opt_cmd.default_value.value, rule_val_map) + + if opt_cmd.required != opt.required: + rule_cmd_type[_RULE_REQUIRE] = " PARSE_RULE_OPTIONAL_%sREQUIRED()" % ("" if opt_cmd.required else "NOT_") + + # A command with any rule of its own gets every rule, since the group it ends up in replaces the option rules entirely + if len(rule_cmd_type) > 0: + rule_cmd[opt_cmd.name] = [ + rule_cmd_type.get(rule, rule_default.get(rule)) + for rule in _RULE_LIST + if rule in rule_cmd_type or rule in rule_default + ] + + if len(rule_cmd) > 0 or len(rule_default) > 0: + result += "\n PARSE_RULE_OPTIONAL\n (\n" + + if len(rule_cmd) > 0: + # Commands whose rules came out the same share one group, which is why the rules are grouped by their text + combine = {} + + for cmd_name, rule_list in rule_cmd.items(): + combine.setdefault("".join("\n%s,\n" % rule for rule in rule_list), []).append(cmd_name) + + for group_idx, (group, cmd_name_list) in enumerate(combine.items()): + if group_idx != 0: + result += "\n" + + result += " PARSE_RULE_OPTIONAL_GROUP\n (\n PARSE_RULE_FILTER_CMD\n (\n" + result += "".join(" PARSE_RULE_VAL_CMD(%s),\n" % bld_enum("", cmd) for cmd in cmd_name_list) + result += " ),\n%s ),\n" % group + + if len(rule_default) > 0: + if len(rule_cmd) > 0: + result += "\n" + + result += " PARSE_RULE_OPTIONAL_GROUP\n (\n" + + for rule_idx, rule in enumerate(rule_default.values()): + result += ("\n" if rule_idx != 0 else "") + "%s,\n" % rule + + result += " ),\n" + + result += " ),\n" + + return result + " )," + + +#################################################################################################################################### +def _render_parse_auto_c(bld_cfg, label): + """Render parse.auto.c.inc, which is the rules the command line and configuration file are parsed with.""" + + rule_val_map = {} + dynamic_default_list = [] + + # Command rules + result = "\n" + bld_comment_block("Command parse data") + result += ( + bld_define("PARSE_RULE_VAL_CMD(value)", "PARSE_RULE_U32_%u(cfgCmd##value)" % _var_128_size(len(bld_cfg.cmd_list) - 1)) + + "\n" + ) + result += "\nstatic const ParseRuleCommand parseRuleCommand[CFG_COMMAND_TOTAL] =\n{\n" + + for cmd_idx, cmd in enumerate(bld_cfg.cmd_list): + if cmd_idx != 0: + result += COMMENT_SEPARATOR + "\n" + + rule = ' PARSE_RULE_COMMAND\n (\n PARSE_RULE_COMMAND_NAME("%s"),\n' % cmd.name + + if cmd.internal: + rule += " PARSE_RULE_COMMAND_INTERNAL(true)\n" + + if cmd.lock_required: + rule += " PARSE_RULE_COMMAND_LOCK_REQUIRED(true),\n" + + if cmd.lock_remote_required: + rule += " PARSE_RULE_COMMAND_LOCK_REMOTE_REQUIRED(true),\n" + + rule += " PARSE_RULE_COMMAND_LOCK_TYPE(%s),\n" % bld_enum("", cmd.lock_type) + + if cmd.log_file: + rule += " PARSE_RULE_COMMAND_LOG_FILE(true),\n" + + rule += " PARSE_RULE_COMMAND_LOG_LEVEL_DEFAULT(%s),\n" % bld_enum("", cmd.log_level_default) + + if cmd.parameter_allowed: + rule += " PARSE_RULE_COMMAND_PARAMETER_ALLOWED(true),\n" + + rule += "\n PARSE_RULE_COMMAND_ROLE_VALID_LIST\n (\n" + rule += "".join(" PARSE_RULE_COMMAND_ROLE(%s)\n" % bld_enum("", role) for role in cmd.role_list) + rule += " ),\n )," + + result += _label(rule, label, "cmd/%s" % cmd.name) + "\n" + + result += "};\n" + + # Option group rules + result += "\n" + bld_comment_block("Option group parse data") + result += "static const ParseRuleOptionGroup parseRuleOptionGroup[CFG_OPTION_GROUP_TOTAL] =\n{\n" + + for opt_grp_idx, opt_grp in enumerate(bld_cfg.opt_grp_list): + if opt_grp_idx != 0: + result += COMMENT_SEPARATOR + "\n" + + rule = ' PARSE_RULE_OPTION_GROUP\n (\n PARSE_RULE_OPTION_GROUP_NAME("%s"),\n ),' % opt_grp.name + + result += _label(rule, label, "opt-grp/%s" % opt_grp.name) + "\n" + + result += "};\n" + + # Option rules + result += "\n" + bld_comment_block("Option parse data") + result += ( + bld_define("PARSE_RULE_VAL_OPT(value)", "PARSE_RULE_U32_%u(cfgOpt##value)" % _var_128_size(len(bld_cfg.opt_list) - 1)) + + "\n" + ) + result += "\nstatic const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =\n{\n" + + for opt_idx, opt in enumerate(bld_cfg.opt_list): + if opt_idx != 0: + result += COMMENT_SEPARATOR + "\n" + + rule = _render_option(opt, bld_cfg, rule_val_map, dynamic_default_list) + + result += _label(rule, label, "opt/%s" % opt.name) + "\n" + + result += "};\n" + + # Option deprecations, which are the old names an option can still be given by + deprecate_list = [] + + for opt in bld_cfg.opt_list: + for deprecate in opt.deprecate_list or []: + deprecate_list.append((deprecate, opt)) + + deprecate_list.sort(key=lambda entry: entry[0].name) + + result += "\n" + bld_comment_block("Option deprecations") + result += bld_define("CFG_OPTION_DEPRECATE_TOTAL", "%u" % len(deprecate_list)) + "\n" + result += "\nstatic const ParseRuleOptionDeprecate parseRuleOptionDeprecate[CFG_OPTION_DEPRECATE_TOTAL] =\n{\n" + + for deprecate_idx, (deprecate, opt) in enumerate(deprecate_list): + if deprecate_idx != 0: + result += COMMENT_SEPARATOR + "\n" + + rule = ' {\n .name = "%s",\n .id = %s,\n' % (deprecate.name, bld_enum("cfgOpt", opt.name)) + + if deprecate.indexed: + rule += " .indexed = true,\n" + + if deprecate.unindexed: + rule += " .unindexed = true,\n" + + rule += " }," + + result += _label(rule, label, "opt-deprecate/%s" % opt.name) + "\n" + + result += "};\n" + + # Order the options are resolved in, since an option cannot be resolved before what it depends on + result += "\n" + bld_comment_block("Order for option parse resolution") + result += "static const uint8_t optionResolveOrder[] =\n{\n" + + resolve_list = [" %s," % bld_enum("cfgOpt", opt.name) for opt in bld_cfg.opt_resolve_list] + + result += _label("\n".join(resolve_list), label, "opt-resolve-order") + "\n};\n" + + # Values the rules refer to, which are collected as the rules are rendered and so come after them + value = _value_render(OPT_TYPE_STRING, rule_val_map, label, "StringPubConst", "String", "Str", "STR", "val/str") + value += _value_render(OPT_TYPE_INTEGER, rule_val_map, label, "int", "Int", "Int", "INT", "val/int") + value += _value_render(OPT_TYPE_SIZE, rule_val_map, label, "int64_t", "Size", "Size", "SIZE", "val/size") + value += _value_render(OPT_TYPE_TIME, rule_val_map, label, "unsigned int", "Time", "Time", "TIME", "val/time") + + value += "\n" + bld_comment_block("Dynamic default values") + value += ( + bld_define( + "PARSE_RULE_DEFAULT_DYNAMIC(value)", + "PARSE_RULE_U32_%u(parseRuleDefaultDynamic##value)" % _var_128_size(len(dynamic_default_list) - 1), + ) + + "\n" + ) + value += "\ntypedef enum\n{\n" + value += "".join(" parseRuleDefaultDynamic%s,\n" % bld_enum("", name) for name in dynamic_default_list) + value += "} ParseRuleDefaultDynamic;\n" + + return bld_header(_MODULE, _PARSE_DESCRIPTION) + value + result + + +#################################################################################################################################### +def bld_cfg_render(path_build, bld_cfg, label): + """Render the configuration files.""" + + file_write_differs(os.path.join(path_build, "src/config/config.auto.h"), _render_config_auto_h(bld_cfg)) + file_write_differs(os.path.join(path_build, "src/config/parse.auto.c.inc"), _render_parse_auto_c(bld_cfg, label)) diff --git a/build/lib/error/parse.py b/build/lib/error/parse.py new file mode 100644 index 000000000..e6657b830 --- /dev/null +++ b/build/lib/error/parse.py @@ -0,0 +1,70 @@ +"""Parse Error Yaml. + +Reads the error declarations, which are the definition of record for every error the C can throw. The order they are declared in is +the order they are rendered in, since the generated array is indexed by nothing but position.""" + +#################################################################################################################################### +import os + +from common.error import ToolError, check +from common.storage import file_read +from common.yaml import yaml_bool, yaml_load + +# Codes an error may use. The range is left open at both ends for the codes the project reserves outside it, e.g. the exit statuses +# the commands themselves return. +CODE_MIN = 25 +CODE_MAX = 125 + + +#################################################################################################################################### +class BldErrError: + """An error type.""" + + def __init__(self, name, code, fatal): + self.name = name + self.code = code + self.fatal = fatal # Is the error fatal, i.e. does it mean the code itself is wrong? + + +#################################################################################################################################### +def _code_parse(value, name): + """Parse and range check an error code.""" + + try: + result = int(value) + except ValueError: + raise ToolError("error '%s' code '%s' is not an integer" % (name, value)) + + check(CODE_MIN <= result <= CODE_MAX, "error '%s' code must be >= %u and <= %u" % (name, CODE_MIN, CODE_MAX)) + + return result + + +#################################################################################################################################### +def bld_err_parse(path_repo): + """Parse error.yaml into the list of error types.""" + + path = os.path.join(path_repo, "build/error.yaml") + result = [] + + for name, detail in yaml_load(file_read(path), path): + code = None + fatal = False + + # A scalar is the code on its own, else a map that may also say whether the error is fatal + if isinstance(detail, str): + code = detail + else: + for key, value in detail: + if key == "code": + code = value + elif key == "fatal": + fatal = yaml_bool(value, "error '%s' fatal" % name) + else: + raise ToolError("unknown error definition '%s'" % key) + + check(code is not None, "error '%s' requires a code" % name) + + result.append(BldErrError(name, _code_parse(code, name), fatal)) + + return result diff --git a/build/lib/error/render.py b/build/lib/error/render.py new file mode 100644 index 000000000..5eedf508e --- /dev/null +++ b/build/lib/error/render.py @@ -0,0 +1,62 @@ +"""Render Error Data. + +Writes the error type declarations and definitions. The definitions include the list the C indexes by code, which is why an error can +be removed from the declaration but its code should not be reused.""" + +#################################################################################################################################### +import os + +from common.render import bld_comment_block, bld_enum, bld_header +from common.storage import file_write_differs + +_MODULE = "error" +_DESCRIPTION = "Error Type Definition" + + +#################################################################################################################################### +def _err_name(name): + """Build the C name of an error, e.g. "file-open" becomes "FileOpenError".""" + + return bld_enum("", name) + "Error" + + +#################################################################################################################################### +def _render_error_auto_h(err_list): + """Render error.auto.h, which declares every error type.""" + + result = bld_header(_MODULE, _DESCRIPTION) + "#ifndef COMMON_ERROR_ERROR_AUTO_H\n#define COMMON_ERROR_ERROR_AUTO_H\n" + + result += "\n" + bld_comment_block("Error type declarations") + + for err in err_list: + result += "ERROR_DECLARE(%s);\n" % _err_name(err.name) + + return result + "\n#endif\n" + + +#################################################################################################################################### +def _render_error_auto_c(err_list): + """Render error.auto.c.inc, which defines every error type and the array the C looks them up in.""" + + result = bld_header(_MODULE, _DESCRIPTION) + + result += "\n" + bld_comment_block("Error type definitions") + + for err in err_list: + result += "ERROR_DEFINE(%3u, %s, %s, RuntimeError);\n" % (err.code, _err_name(err.name), "true" if err.fatal else "false") + + result += "\n" + bld_comment_block("Error type array") + result += "static const ErrorType *const errorTypeList[] =\n{\n" + + for err in err_list: + result += " &%s,\n" % _err_name(err.name) + + return result + " NULL,\n};\n" + + +#################################################################################################################################### +def bld_err_render(path_build, err_list): + """Render the error files.""" + + file_write_differs(os.path.join(path_build, "src/common/error/error.auto.h"), _render_error_auto_h(err_list)) + file_write_differs(os.path.join(path_build, "src/common/error/error.auto.c.inc"), _render_error_auto_c(err_list)) diff --git a/build/lib/help/parse.py b/build/lib/help/parse.py new file mode 100644 index 000000000..c2eabae64 --- /dev/null +++ b/build/lib/help/parse.py @@ -0,0 +1,243 @@ +"""Parse Help Xml. + +Reads the help text for every command and option. The configuration declaration says what exists and this says what it means, so the +two are checked against each other here: a command or an option the user can reach must have help, and help for something that does not +exist is a leftover. + +The help text is left as xml nodes rather than rendered, since the same nodes are rendered as console text for the help and as +documentation for the reference.""" + +#################################################################################################################################### +from common.error import ToolError, check +from common.storage import file_read +from common.xml import xml_node_attribute, xml_node_child, xml_node_child_list, xml_node_content, xml_parse +from config.parse import CMD_ROLE_MAIN + + +#################################################################################################################################### +class BldHlpSection: + """A section of the configuration, i.e. a group of options that are documented together.""" + + def __init__(self, id, name, introduction): + self.id = id + self.name = name + self.introduction = introduction + + +#################################################################################################################################### +class BldHlpCommand: + """Help for a command.""" + + def __init__(self, name, title, summary, description, opt_list): + self.name = name + self.title = title + self.summary = summary + self.description = description + self.opt_list = opt_list # Help for the options this command documents differently, if any + + +#################################################################################################################################### +class BldHlpOption: + """Help for an option.""" + + def __init__(self, name, section, title, summary, description, example_list): + self.name = name + self.section = section + self.title = title + self.summary = summary + self.description = description + self.example_list = example_list + + +#################################################################################################################################### +class BldHlp: + """The help declaration.""" + + def __init__(self): + self.sct_list = None # Configuration sections + self.cmd_title = None + self.cmd_description = None + self.cmd_introduction = None + self.cmd_list = None + self.opt_title = None + self.opt_description = None + self.opt_introduction = None + self.opt_list = None + + +#################################################################################################################################### +def _content(node): + """Text a node holds, or None when the node is not there because it was not required.""" + + return None if node is None else xml_node_content(node) + + +#################################################################################################################################### +def _find(item_list, name): + """Find an entry in a list by name, or None when it is not there.""" + + for item in item_list or []: + if item.name == name: + return item + + return None + + +#################################################################################################################################### +def _option_list(xml_opt_list, opt_list, section_default): + """Parse a list of option help into opt_list.""" + + for xml_opt in xml_opt_list: + example_list = [xml_node_content(example) for example in xml_node_child_list(xml_opt, "example")] + + opt_list.append( + BldHlpOption( + xml_node_attribute(xml_opt, "id", True), + xml_node_attribute(xml_opt, "section") or section_default, + xml_node_attribute(xml_opt, "name", True), + xml_node_child(xml_opt, "summary", True), + xml_node_child(xml_opt, "text", True), + example_list if len(example_list) > 0 else None, + ) + ) + + opt_list.sort(key=lambda opt: opt.name) + + +#################################################################################################################################### +def _config_section_list(xml): + """The configuration sections, which are where the options that can be set in the configuration file are documented.""" + + return xml_node_child_list(xml_node_child(xml_node_child(xml, "config", True), "config-section-list", True), "config-section") + + +#################################################################################################################################### +def _help_option_list(xml): + """Parse the help for every option, wherever it is documented.""" + + result = [] + + # Options that can be set in the configuration file, which take the section they are documented in + for xml_section in _config_section_list(xml): + _option_list( + xml_node_child_list(xml_node_child(xml_section, "config-key-list", True), "config-key"), + result, + xml_node_attribute(xml_section, "id", True), + ) + + # Options that can only be given on the command line, which are in no section + _option_list( + xml_node_child_list( + xml_node_child(xml_node_child(xml_node_child(xml, "operation", True), "operation-general", True), "option-list", True), + "option", + ), + result, + None, + ) + + return result + + +#################################################################################################################################### +def _help_command_list(xml): + """Parse the help for every command, and for the options a command documents differently.""" + + result = [] + + for xml_cmd in xml_node_child_list(xml, "command"): + opt_list = None + xml_opt_list = xml_node_child(xml_cmd, "option-list") + + if xml_opt_list is not None: + opt_list = [] + _option_list(xml_node_child_list(xml_opt_list, "option"), opt_list, None) + + result.append( + BldHlpCommand( + xml_node_attribute(xml_cmd, "id", True), + xml_node_attribute(xml_cmd, "name", True), + xml_node_child(xml_cmd, "summary", True), + xml_node_child(xml_cmd, "text", True), + opt_list, + ) + ) + + result.sort(key=lambda cmd: cmd.name) + + return result + + +#################################################################################################################################### +def _help_section_list(xml, detail): + """Parse the configuration sections.""" + + result = [ + BldHlpSection( + xml_node_attribute(xml_section, "id", True), + xml_node_attribute(xml_section, "name", True), + xml_node_child(xml_section, "text", detail), + ) + for xml_section in _config_section_list(xml) + ] + + result.sort(key=lambda section: section.id) + + return result + + +#################################################################################################################################### +def _validate(bld_hlp, bld_cfg): + """Check the help against the configuration, since anything the user can reach needs help.""" + + for cmd in bld_cfg.cmd_list: + if _find(bld_hlp.cmd_list, cmd.name) is None: + raise ToolError("command '%s' must have help" % cmd.name) + + for opt in bld_cfg.opt_list: + if _find(bld_hlp.opt_list, opt.name) is not None: + continue + + # An option documented under neither a configuration section nor the command line must be documented by every command that + # takes it, since that is the only place left for its help to be + for opt_cmd in opt.cmd_list: + cmd_hlp = _find(bld_hlp.cmd_list, opt_cmd.name) + check(cmd_hlp is not None, "command help for '%s' is missing" % opt_cmd.name) + + # Only an option a user can give needs help, i.e. one the main role of the command takes + if CMD_ROLE_MAIN not in opt_cmd.role_list: + continue + + if _find(cmd_hlp.opt_list, opt.name) is None: + raise ToolError("option '%s' must have help for command '%s'" % (opt.name, opt_cmd.name)) + + +#################################################################################################################################### +def bld_hlp_parse(path_help, bld_cfg, detail): + """Parse the help declaration. + + The path is passed in rather than built from the repository, as the other declarations are, because this one belongs to the + documentation rather than to the build and so is not the build library's to know about. + + Detail says whether the text that only the documentation renders is required, since the help in the binary does not use it.""" + + xml = xml_parse(file_read(path_help), path_help) + + xml_cfg = xml_node_child(xml, "config", True) + xml_operation = xml_node_child(xml, "operation", True) + + result = BldHlp() + result.sct_list = _help_section_list(xml, detail) + + result.cmd_title = xml_node_attribute(xml_operation, "title", True) + result.cmd_description = _content(xml_node_child(xml_operation, "description", detail)) + result.cmd_introduction = xml_node_child(xml_operation, "text", detail) + result.cmd_list = _help_command_list(xml_node_child(xml_operation, "command-list", True)) + + result.opt_title = xml_node_attribute(xml_cfg, "title", True) + result.opt_description = _content(xml_node_child(xml_cfg, "description", detail)) + result.opt_introduction = xml_node_child(xml_cfg, "text", detail) + result.opt_list = _help_option_list(xml) + + _validate(result, bld_cfg) + + return result diff --git a/build/lib/help/render.py b/build/lib/help/render.py new file mode 100644 index 000000000..ef8bbeabb --- /dev/null +++ b/build/lib/help/render.py @@ -0,0 +1,263 @@ +"""Render Help. + +Writes the help the binary carries for itself. The text is rendered from the same xml the documentation is built from, packed, and +compressed, so the help for every command and option costs about as much as one page of it would uncompressed. + +The result is a C string literal rather than a list of byte values, which is far more compact, and the data is not terminated so the +array is declared as not being a string and sized to exactly what is there.""" + +#################################################################################################################################### +import bz2 +import os + +from common.error import ToolError +from common.pack import PackWrite +from common.render import LINE_LENGTH, bld_header +from common.storage import file_write, file_write_differs + +_MODULE = "help" +_DESCRIPTION = "Help Data" + +# Compression level, which is the most bzip2 offers since the data is compressed once and read forever +_COMPRESS_LEVEL = 9 + +# Tags that render as the text they stand for +_TAG_TEXT = {"backrest": "pgBackRest", "postgres": "PostgreSQL"} + +# Tags that render as what they contain, followed by what they add. A tag that adds nothing renders as its content alone. +_TAG_WRAP = { + "admonition": ("NOTE: ", "\n\n"), + "list": ("", "\n"), + "list-item": ("* ", "\n"), + "p": ("", "\n\n"), + "quote": ('"', '"'), +} + +# Tags that render as what they contain +_TAG_CONTENT = ( + "b", + "br-option", + "cmd", + "code", + "exe", + "file", + "host", + "i", + "id", + "link", + "path", + "pg-setting", + "proper", + "setting", +) + +# Bytes that have a short escape in C, which are written as that rather than as an octal escape +_ESCAPE = { + 0x07: "\\a", + 0x08: "\\b", + 0x09: "\\t", + 0x0A: "\\n", + 0x0B: "\\v", + 0x0C: "\\f", + 0x0D: "\\r", + ord('"'): '\\"', + ord("\\"): "\\\\", + ord("?"): "\\?", +} + + +#################################################################################################################################### +def _render_text(text): + """Render a run of text. + + Text that runs over more than one line is how the xml is laid out to be read rather than something to render, so it is dropped + once it is known to be only whitespace.""" + + if text is None: + return "" + + if "\n" in text: + if text.strip() != "": + raise ToolError("text '%s' is invalid" % text) + + return "" + + # The dash is written as a variable in the xml so it does not get treated as markup by the documentation renderers + return text.replace("{[dash]}", "-") + + +#################################################################################################################################### +def _render_node(node): + """Render the content of a node as console text.""" + + result = _render_text(node.text) + + for child in node: + if child.tag in _TAG_TEXT: + result += _TAG_TEXT[child.tag] + elif child.tag in _TAG_WRAP: + begin, end = _TAG_WRAP[child.tag] + result += begin + _render_node(child) + end + elif child.tag in _TAG_CONTENT: + result += _render_node(child) + else: + raise ToolError("unknown tag '%s'" % child.tag) + + result += _render_text(child.tail) + + return result + + +#################################################################################################################################### +def bld_hlp_render_xml(node): + """Render a help node as the console text the help shows.""" + + return _render_node(node).strip() + + +#################################################################################################################################### +def _find(item_list, name): + """Find an entry in a list by name, or None when it is not there.""" + + for item in item_list or []: + if item.name == name: + return item + + return None + + +#################################################################################################################################### +def _pack(bld_cfg, bld_hlp): + """Pack the help, which is one entry per command and one per option, in the order the code looks them up in.""" + + pack = PackWrite() + + pack.array_begin() + + for cmd in bld_cfg.cmd_list: + cmd_hlp = _find(bld_hlp.cmd_list, cmd.name) + + pack.bool_write(cmd.internal) + pack.str_write(bld_hlp_render_xml(cmd_hlp.summary)) + pack.str_write(bld_hlp_render_xml(cmd_hlp.description)) + + pack.array_end() + + pack.array_begin() + + for opt in bld_cfg.opt_list: + opt_hlp = _find(bld_hlp.opt_list, opt.name) + + pack.bool_write(opt.internal) + + # An option documented only by the commands that take it has no help of its own here + pack.str_write(opt_hlp.section if opt_hlp is not None else None) + pack.str_write(bld_hlp_render_xml(opt_hlp.summary) if opt_hlp is not None else None) + pack.str_write(bld_hlp_render_xml(opt_hlp.description) if opt_hlp is not None else None) + + # Deprecated names, less any that is the option name itself since that is not a name to report as deprecated + deprecate_list = [deprecate.name for deprecate in opt.deprecate_list or [] if deprecate.name != opt.name] + + if len(deprecate_list) > 0: + pack.array_begin() + + for name in deprecate_list: + pack.str_write(name) + + pack.array_end() + else: + pack.null_write() + + # What a command says about the option that differs from what the option says, i.e. its own help or being internal + found = False + + for cmd_idx, cmd in enumerate(bld_cfg.cmd_list): + opt_cmd = _find(opt.cmd_list, cmd.name) + + if opt_cmd is None: + continue + + cmd_opt_hlp = _find(_find(bld_hlp.cmd_list, cmd.name).opt_list, opt.name) + + if opt.internal == opt_cmd.internal and cmd_opt_hlp is None: + continue + + if not found: + pack.array_begin() + found = True + + pack.obj_begin(id=cmd_idx + 1) + + if opt.internal != opt_cmd.internal: + pack.bool_write(opt_cmd.internal, default_write=True) + else: + pack.null_write() + + if cmd_opt_hlp is not None: + pack.str_write(bld_hlp_render_xml(cmd_opt_hlp.summary)) + pack.str_write(bld_hlp_render_xml(cmd_opt_hlp.description)) + + pack.obj_end() + + if found: + pack.array_end() + else: + pack.null_write() + + pack.array_end() + + return pack.end() + + +#################################################################################################################################### +def _bld_hlp_data(bld_cfg, bld_hlp): + """The packed and compressed help, which is what the binary carries and decompresses at run time.""" + + return bz2.compress(_pack(bld_cfg, bld_hlp), _COMPRESS_LEVEL) + + +#################################################################################################################################### +def _render_help_auto_c(bld_cfg, bld_hlp): + """Render help.auto.c.inc, which is the compressed help as a C string literal.""" + + data = _bld_hlp_data(bld_cfg, bld_hlp) + + result = bld_header(_MODULE, _DESCRIPTION) + result += 'VR_NON_STRING static const char helpData[%u] =\n"' % len(data) + + # Break the physical lines at a fixed width with a continuation, which the preprocessor removes before the string is tokenized. + # Breaking here rather than between escapes keeps every line the same width even when the break falls inside an escape sequence. + # The opening quote takes the first column of the first line. + line = [""] + line_size = 1 + + for byte in data: + escape = _ESCAPE.get(byte, chr(byte) if 0x20 <= byte <= 0x7E else "\\%03o" % byte) + + for char in escape: + # Counting the continuation the line is as wide as the rest of the source + if line_size == LINE_LENGTH - 1: + line.append("") + line_size = 0 + + line[-1] += char + line_size += 1 + + return result + "\\\n".join(line) + '";\n' + + +#################################################################################################################################### +def bld_hlp_render(path_build, bld_cfg, bld_hlp): + """Render the help.""" + + file_write_differs(os.path.join(path_build, "src/command/help/help.auto.c.inc"), _render_help_auto_c(bld_cfg, bld_hlp)) + + +#################################################################################################################################### +def bld_hlp_render_data(path_build, bld_cfg, bld_hlp): + """Write the help as raw data rather than as C. + + The help unit test loads this to run the help against the current declarations, since the C literal the build compiles in may not + have been generated when the test runs. It goes at the root of the build path because it is not part of any source tree.""" + + file_write(os.path.join(path_build, "help.dat"), _bld_hlp_data(bld_cfg, bld_hlp)) diff --git a/build/lib/postgres/parse.py b/build/lib/postgres/parse.py new file mode 100644 index 000000000..1bf1e6e0f --- /dev/null +++ b/build/lib/postgres/parse.py @@ -0,0 +1,144 @@ +"""Parse PostgreSQL Interface Declaration. + +The versions come from postgres.yaml, but the types, defines, and functions that make up an interface are read out of the vendored +PostgreSQL headers rather than declared. A version interface is the same headers compiled again with different names, so scanning +them is what keeps the generated code in step with a header update.""" + +#################################################################################################################################### +import os + +from common.error import ToolError +from common.storage import file_read +from common.yaml import yaml_bool, yaml_load + +# Defines that every interface undefines but that no vendored header declares, so they are added to the list explicitly +_DEFINE_EXTRA_LIST = ("CATALOG_VERSION_NO_MAX", "PG_VERSION") + + +#################################################################################################################################### +class BldPgVersion: + """A supported PostgreSQL version.""" + + def __init__(self, version, release): + self.version = version + self.release = release # Has the version been released? + + +#################################################################################################################################### +class BldPg: + """The PostgreSQL interface declaration.""" + + def __init__(self, pg_list, type_list, define_list, function_list): + self.pg_list = pg_list # Supported versions, oldest first + self.type_list = type_list # Interface types, sorted + self.define_list = define_list # Interface defines, sorted + self.function_list = function_list # Functions defined by macros, in the order they are declared + + +#################################################################################################################################### +def _version_list(path): + """Parse the version list.""" + + result = [] + + for key, value in yaml_load(file_read(path), path): + if key != "version": + raise ToolError("unknown postgres definition '%s'" % key) + + for version in value: + # A scalar is the version on its own, else a map naming the version and its attributes + if isinstance(version, str): + result.append(BldPgVersion(version, True)) + + continue + + for name, detail in version: + release = True + + for def_key, def_value in detail: + if def_key != "release": + raise ToolError("unknown postgres definition '%s'" % def_key) + + release = yaml_bool(def_value, "version '%s' release" % name) + + result.append(BldPgVersion(name, release)) + + return result + + +#################################################################################################################################### +def _define_list(header): + """Scan the defines out of a header.""" + + result = [] + + for line in header.split("\n"): + line = line.strip() + + if not line.startswith("#define"): + continue + + token = line.split(" ")[1].strip() + + if token == "": + raise ToolError("unable to find define -- are there extra spaces on '%s'" % line) + + # The define name may be followed by a parameter list or separated from its value by a tab + define = token.split("(")[0] if "(" in token else token.split("\t")[0] + define = define.strip() + + if define not in result: + result.append(define) + + return result + + +#################################################################################################################################### +def _type_list(header): + """Scan the types out of a header. + + A typedef of a struct or an enum names the type after the block rather than before it, so the scan carries on to the closing + brace, collecting the values of an enum on the way since each is a name the interface has to undefine.""" + + result = [] + scan_enum = False + + def add(value): + if value not in result: + result.append(value) + + for line in header.split("\n"): + token_list = line.strip().split(" ") + + if token_list[0] == "typedef": + # A struct or an enum is named at the end of the block, so keep scanning + if token_list[1] in ("struct", "enum"): + scan_enum = token_list[1] == "enum" + else: + add(token_list[-1].split(";")[0]) + elif token_list[0] == "}": + add(token_list[-1].split(";")[0]) + scan_enum = False + elif scan_enum and token_list[0] != "{": + add(token_list[0].split(",")[0]) + + return result + + +#################################################################################################################################### +def bld_pg_parse(path_repo): + """Parse the PostgreSQL interface declaration.""" + + path_vendor = os.path.join(path_repo, "src/postgres/interface/version.vendor.h") + header_vendor = file_read(path_vendor) + + # The interface is generated from the vendored header, so its types and defines are whatever that header declares + type_list = sorted(_type_list(header_vendor)) + define_list = sorted(_define_list(header_vendor) + list(_DEFINE_EXTRA_LIST)) + + # Functions are defined as macros, which each interface expands for its own version + function_list = _define_list(file_read(os.path.join(path_repo, "src/postgres/interface/version.intern.h"))) + + path_yaml = os.path.join(path_repo, "build/postgres.yaml") + + return BldPg(_version_list(path_yaml), type_list, define_list, function_list) diff --git a/build/lib/postgres/render.py b/build/lib/postgres/render.py new file mode 100644 index 000000000..bf20f9aff --- /dev/null +++ b/build/lib/postgres/render.py @@ -0,0 +1,135 @@ +"""Render PostgreSQL Interface. + +Each version gets a block that includes the vendored header again with every name suffixed by the version, which is how one binary +carries an interface for every version it supports. The interfaces are rendered newest first so the most likely match is found first +at run time.""" + +#################################################################################################################################### +import os + +from common.render import bld_comment_block, bld_define, bld_header +from common.storage import file_write_differs + +_MODULE = "postgres" +_DESCRIPTION = "PostgreSQL Interface" + +_VERSION_MODULE = "postgres-version" +_VERSION_DESCRIPTION = "PostgreSQL Version" + + +#################################################################################################################################### +def _version_no_dot(version): + """The version as it appears in a C name, e.g. "9.6" becomes "96".""" + + return version.replace(".", "") + + +#################################################################################################################################### +def _version_num(version): + """The version as the number the C compares, e.g. "9.6" becomes 90600 and "10" becomes 100000.""" + + major, _, minor = version.partition(".") + + return int(major) * 10000 + (int(minor) * 100 if minor != "" else 0) + + +#################################################################################################################################### +def _function_name(define): + """The interface function a macro defines, e.g. "PG_INTERFACE_CONTROL_IS" becomes "pgInterfaceControlIs".""" + + result = "" + + for idx, part in enumerate(define.split("_")): + part = part.lower() + result += part[:1].upper() + part[1:] if idx != 0 else part + + return result + + +#################################################################################################################################### +def _render_interface_auto_c(bld_pg): + """Render interface.auto.c.inc, which is one interface per version plus the struct that finds them.""" + + result = bld_header(_MODULE, _DESCRIPTION) + + # Interfaces, newest first + for pg in reversed(bld_pg.pg_list): + version_no_dot = _version_no_dot(pg.version) + + result += "\n" + bld_comment_block("PostgreSQL %s interface" % pg.version) + result += bld_define("PG_VERSION", "PG_VERSION_%s" % version_no_dot) + "\n\n" + + for type in bld_pg.type_list: + result += bld_define(type, "%s_%s" % (type, version_no_dot)) + "\n" + + # An unreleased version has no catalog version of its own yet, so it accepts any catalog version up to the maximum + if not pg.release: + result += "\n#define CATALOG_VERSION_NO_MAX\n" + + result += '\n#include "postgres/interface/version.intern.h"\n\n' + + for function in bld_pg.function_list: + result += "%s(%s);\n" % (function, version_no_dot) + + # Undefine everything the interface defined so the next one starts clean + result += "\n" + "".join("#undef %s\n" % type for type in bld_pg.type_list) + result += "\n" + "".join("#undef %s\n" % define for define in bld_pg.define_list) + result += "\n" + "".join("#undef %s\n" % function for function in bld_pg.function_list) + + # Interface struct, newest first so the most likely match is found first + result += "\n" + bld_comment_block("PostgreSQL interface struct") + result += "static const PgInterface pgInterface[] =\n{\n" + + for pg in reversed(bld_pg.pg_list): + version_no_dot = _version_no_dot(pg.version) + + result += " {\n .version = PG_VERSION_%s,\n\n" % version_no_dot + + for function in bld_pg.function_list: + name = _function_name(function) + member = name[len("pgInterface") :] + + result += " .%s = %s%s,\n" % (member[:1].lower() + member[1:], name, version_no_dot) + + result += " },\n" + + return result + "};\n" + + +#################################################################################################################################### +def _render_version_auto_h(bld_pg): + """Render version.auto.h, which is the version numbers and the strings the errors report them with.""" + + result = bld_header(_VERSION_MODULE, _VERSION_DESCRIPTION) + result += "#ifndef POSTGRES_VERSION_AUTO_H\n#define POSTGRES_VERSION_AUTO_H\n" + + result += "\n" + bld_comment_block("PostgreSQL version constants") + + for pg in bld_pg.pg_list: + version_no_dot = _version_no_dot(pg.version) + + result += bld_define("PG_VERSION_%s" % version_no_dot, "%u" % _version_num(pg.version)) + "\n" + + # The newest version is the maximum, which the code uses to reject anything newer than it knows about + result += "\n" + bld_define("PG_VERSION_MAX", "PG_VERSION_%s" % _version_no_dot(bld_pg.pg_list[-1].version)) + "\n" + + result += "\n" + bld_comment_block("PostgreSQL version string constants for use in error messages") + + for pg in bld_pg.pg_list: + result += bld_define("PG_VERSION_%s_Z" % _version_no_dot(pg.version), '"%s"' % pg.version) + "\n" + + return result + "\n#endif\n" + + +#################################################################################################################################### +def bld_pg_render(path_build, bld_pg): + """Render the PostgreSQL interfaces.""" + + file_write_differs(os.path.join(path_build, "src/postgres/interface.auto.c.inc"), _render_interface_auto_c(bld_pg)) + + +#################################################################################################################################### +def bld_pg_version_render(path_build, bld_pg): + """Render the PostgreSQL version constants.""" + + file_write_differs(os.path.join(path_build, "src/postgres/version.auto.h"), _render_version_auto_h(bld_pg)) diff --git a/src/build/postgres/postgres.yaml b/build/postgres.yaml similarity index 100% rename from src/build/postgres/postgres.yaml rename to build/postgres.yaml diff --git a/doc/doc.pl b/doc/doc.pl index dc8abc769..d1aaecf76 100755 --- a/doc/doc.pl +++ b/doc/doc.pl @@ -217,19 +217,10 @@ eval $rhVariableOverride->{$strKey} = $rhKeyVariableOverride->{$strKey}; } - # Build C code - my $strBuildPath = "${strBasePath}/output/build"; + # Build the reference documents, the user guide, and the manual page my $strRepoPath = dirname($strBasePath); - my $strBuildNinja = "${strBuildPath}/build.ninja"; my $strBuildVar = ""; - &log(INFO, "build C helper"); - - if (!-e $strBuildNinja) - { - executeTest("meson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug ${strBuildPath} ${strRepoPath}"); - } - foreach my $strVar (sort(keys(%{$rhVariableOverride}))) { $strBuildVar .= " --var=${strVar}=\"" . $rhVariableOverride->{$strVar} . '"'; @@ -237,9 +228,8 @@ eval $strBuildVar .= " --var=debug=" . ($bDebug ? 'y' : 'n'); - executeTest("ninja -C ${strBuildPath} doc/src/doc-pgbackrest"); executeTest( - "${strBuildPath}/doc/src/doc-pgbackrest --repo-path=${strRepoPath}${strBuildVar}" . + "${strBasePath}/doc.py --repo-path=${strRepoPath}${strBuildVar}" . ($strLogLevel ne 'info' ? " --log-level=${strLogLevel}" : ''), {bShowOutputAsync => true}); diff --git a/doc/doc.py b/doc/doc.py new file mode 100755 index 000000000..1e280da41 --- /dev/null +++ b/doc/doc.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""pgBackRest Documentation Builder. + +Builds the reference documents, the user guide, and the manual page from the same declarations the binary is generated from. The +renderers that turn them into html and markdown run separately. + +Exit status: 0 = success, greater = error.""" + +#################################################################################################################################### +import os +import sys + +# Do not cache bytecode. The tool runs from the source tree, where a __pycache__ would show up as an unexpected binary file in the +# linter and in the distribution. This must be set before the library modules are imported below. +sys.dont_write_bytecode = True + +# Each tool keeps its library beside itself and may use the libraries below it in the hierarchy. Insert them first, lowest last, so the +# doc modules are found before anything else on the path. +for lib in ("build", "doc"): + sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), lib, "lib")) + +from command.build.build import cfg_load, cmd_build # noqa: E402 +from common.error import EXIT_ERROR, ToolError # noqa: E402 +from common.log import * # noqa: E402 + + +#################################################################################################################################### +def main(): + """Build the documentation.""" + + path_repo = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + # Load the configuration. Logging is not initialized until this succeeds so print any error directly. + try: + config = cfg_load(sys.argv[1:], path_repo) + except ToolError as error: + print(error) + + return EXIT_ERROR + + log_init(log_level_parse(config.log_level), config.log_timestamp) + log(INFO, "build documentation") + + try: + cmd_build(config.repo_path, config.var_map) + except ToolError as error: + log(ERROR, error) + + return error.status + + log(INFO, "build documentation end: completed successfully") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/doc/lib/command/build/build.py b/doc/lib/command/build/build.py new file mode 100644 index 000000000..66d3df7d7 --- /dev/null +++ b/doc/lib/command/build/build.py @@ -0,0 +1,94 @@ +"""Build Documentation. + +Builds everything the renderers work from: the command and configuration references, the user guide with everything in it resolved, and +the manual page. All four come from the declarations the binary is generated from, so the documentation describes what the code does. + +The variables given on the command line are loaded before the user guide is read, so a document cannot override what the caller asked +for.""" + +#################################################################################################################################### +import argparse +import os + +from command.build.man import reference_man_render +from command.build.pre import build_pre +from command.build.reference import reference_command_render, reference_configuration_render +from command.build.var_store import VarStore +from common.error import ToolError +from common.log import LEVEL_NAME +from common.storage import file_read, file_write +from common.xml import xml_document_parse, xml_parse +from config.parse import bld_cfg_parse +from help.parse import bld_hlp_parse + +# Where the documentation is read from and written to, relative to the repository +_PATH_XML = "doc/xml" +_PATH_OUT_XML = "doc/output/xml" +_PATH_OUT_MAN = "doc/output/man" + +_FILE_MAN = "pgbackrest.1.txt" + + +#################################################################################################################################### +def cfg_load(arg_list, path_repo): + """Parse the command line.""" + + parser = argparse.ArgumentParser(prog="doc.py", description="Build the pgBackRest documentation.") + parser.add_argument("--repo-path", default=path_repo, metavar="PATH", help="code repository path") + parser.add_argument("--var", action="append", default=[], metavar="KEY=VALUE", help="variable to set in the documentation") + parser.add_argument( + "--log-level", default="info", choices=sorted(LEVEL_NAME.values()), metavar="LEVEL", help="console log level" + ) + parser.add_argument("--no-log-timestamp", dest="log_timestamp", action="store_false", help="suppress timestamps in the log") + + result = parser.parse_args(arg_list) + + # A relative path is made absolute so it does not depend on where the tool was run from + result.repo_path = os.path.abspath(result.repo_path) + + # Variables are given as key=value, and a value may itself hold an equals sign + result.var_map = {} + + for var in result.var: + key, _, value = var.partition("=") + + if key == "" or "=" not in var: + raise ToolError("variable '%s' must be given as key=value" % var) + + result.var_map[key] = value + + return result + + +#################################################################################################################################### +def _read(path_repo, path, document=False): + """Read xml from the repository.""" + + path = os.path.join(path_repo, path) + content = file_read(path) + + return xml_document_parse(content, path) if document else xml_parse(content, path) + + +#################################################################################################################################### +def cmd_build(path_repo, var_map): + """Build the documentation.""" + + var_store = VarStore() + + # The caller's variables are loaded first, so a document declaring the same one does not override them + for variable, value in var_map.items(): + var_store.add(variable, value) + + bld_cfg = bld_cfg_parse(path_repo) + bld_hlp = bld_hlp_parse(os.path.join(path_repo, _PATH_XML, "reference.xml"), bld_cfg, True) + + index = _read(path_repo, os.path.join(_PATH_XML, "index.xml")) + user_guide = _read(path_repo, os.path.join(_PATH_XML, "user-guide.xml"), document=True) + + file_write(os.path.join(path_repo, _PATH_OUT_XML, "command.xml"), reference_command_render(bld_cfg, bld_hlp).render()) + file_write( + os.path.join(path_repo, _PATH_OUT_XML, "configuration.xml"), reference_configuration_render(bld_cfg, bld_hlp).render() + ) + file_write(os.path.join(path_repo, _PATH_OUT_XML, "user-guide.xml"), build_pre(user_guide, bld_hlp, var_store).render()) + file_write(os.path.join(path_repo, _PATH_OUT_MAN, _FILE_MAN), reference_man_render(index, bld_cfg, bld_hlp)) diff --git a/doc/lib/command/build/eval.py b/doc/lib/command/build/eval.py new file mode 100644 index 000000000..b3a707868 --- /dev/null +++ b/doc/lib/command/build/eval.py @@ -0,0 +1,115 @@ +"""Expression Evaluation. + +Evaluates the expression an if attribute holds, which decides whether the node it is on stays in the document. Variables are replaced +before the expression gets here, so it works on values rather than on names. + +The grammar is only what the documentation needs: or, and, parentheses, negation, string equality, and numeric comparison. Or binds +loosest and is split first, so "a || b && c" is "a || (b && c)".""" + +#################################################################################################################################### +from common.error import ToolError + +# Numeric comparisons, longest operator first so that >= is not read as > followed by = +_COMPARE_LIST = ( + (" >= ", lambda left, right: left >= right), + (" <= ", lambda left, right: left <= right), + (" > ", lambda left, right: left > right), + (" < ", lambda left, right: left < right), +) + + +#################################################################################################################################### +def _find_op(expression, op): + """Find an operator that is not inside parentheses, since one that is belongs to the expression they hold.""" + + depth = 0 + + for idx in range(len(expression)): + if expression[idx] == "(": + depth += 1 + elif expression[idx] == ")": + depth -= 1 + elif depth == 0 and expression.startswith(op, idx): + return idx + + return -1 + + +#################################################################################################################################### +def _quoted(value, side, op, expression): + """The value a quoted string holds, since a comparison is between what was written rather than how it was written.""" + + if len(value) < 2 or value[0] != "'" or value[-1] != "'": + raise ToolError("expected quoted string on %s side of '%s': '%s'" % (side, op, expression)) + + return value[1:-1] + + +#################################################################################################################################### +def _number(value, expression): + """The number a value holds.""" + + try: + return int(value.strip()) + except ValueError: + raise ToolError("expected number in '%s'" % expression) + + +#################################################################################################################################### +def _eval_one(expression): + """Evaluate an expression that has no or/and in it.""" + + trimmed = expression.strip() + + # An expression in parentheses is evaluated on its own + if trimmed.startswith("("): + if not trimmed.endswith(")"): + raise ToolError("unmatched parenthesis in '%s'" % expression) + + return eval_expression(trimmed[1:-1]) + + if trimmed.startswith("!"): + return not _eval_one(trimmed[1:]) + + # String comparison + for op in ("eq", "ne"): + idx = trimmed.find(" %s " % op) + + if idx != -1: + left = _quoted(trimmed[:idx].strip(), "left", op, expression) + right = _quoted(trimmed[idx + len(op) + 2 :].strip(), "right", op, expression) + + return left == right if op == "eq" else left != right + + # Numeric comparison + for op, compare in _COMPARE_LIST: + idx = trimmed.find(op) + + if idx != -1: + return compare(_number(trimmed[:idx], expression), _number(trimmed[idx + len(op) :], expression)) + + raise ToolError("unable to evaluate '%s'" % expression) + + +#################################################################################################################################### +def eval_expression(expression): + """Evaluate an expression, which must have had its variables replaced already.""" + + if "{[" in expression: + raise ToolError("unreplaced variable in expression '%s'" % expression) + + # Or binds loosest so it is split first, which leaves and to bind tighter than it. Each side is evaluated only when it is needed, + # so an expression that is decided by its left side is not held to what is on its right. + for op in (" || ", " && "): + idx = _find_op(expression, op) + + if idx != -1: + left = expression[:idx].strip() + right = expression[idx + len(op) :].strip() + + if op == " || ": + return eval_expression(left) or eval_expression(right) + + return eval_expression(left) and eval_expression(right) + + return _eval_one(expression) diff --git a/doc/lib/command/build/man.py b/doc/lib/command/build/man.py new file mode 100644 index 000000000..1f1ca6ddc --- /dev/null +++ b/doc/lib/command/build/man.py @@ -0,0 +1,259 @@ +"""Build Manual Page Reference. + +Renders the manual page, which is the reference as plain text for a console rather than as a document. It lists every command and every +option with its summary, wrapped to a fixed width. + +The wrapping is a port of what the binary does for its own help, so the manual page and `pgbackrest help` break lines the same way.""" + +#################################################################################################################################### +from common.error import ToolError, check +from common.xml import xml_node_attribute, xml_node_child, xml_node_content +from help.render import bld_hlp_render_xml + +# Width the text is wrapped to. A fixed 80 is safe on virtually any console. +_CONSOLE_WIDTH = 80 + +# Section an option is listed under when it has none of its own +_SECTION_DEFAULT = "General" + +# Variables the manual page resolves, which is only what it actually uses. Anything else is an error rather than text that would be +# rendered as the name of a variable. +_VAR_MAP = {"{[postgres]}": "PostgreSQL", "{[project]}": "pgBackRest"} + +# What the manual page says about itself, which is the same whatever the declarations hold +_NAME = "pgBackRest" +_BIN = "pgbackrest" +_CONFIG_FILE = "/etc/pgbackrest/pgbackrest.conf" + +_FILES = ( + """ +FILES + %s + /var/lib/pgbackrest + /var/log/pgbackrest + /var/spool/pgbackrest + /tmp/pgbackrest +""" + % _CONFIG_FILE +) + +_EXAMPLES = ( + """ +EXAMPLES + * Create a backup of the PostgreSQL `main` cluster: + + $ pgbackrest --stanza=main backup + + The `main` cluster should be configured in `%s` + + * Show all available backups: + + $ pgbackrest info + + * Show all available backups for a specific cluster: + + $ pgbackrest --stanza=main info + + * Show backup specific options: + + $ pgbackrest help backup +""" + % _CONFIG_FILE +) + +_SEE_ALSO = """ +SEE ALSO + /usr/share/doc/pgbackrest-doc/html/index.html + https://pgbackrest.org +""" + + +#################################################################################################################################### +def _find(item_list, name): + """Find an entry in a list by name, or None when it is not there.""" + + for item in item_list: + if item.name == name: + return item + + return None + + +#################################################################################################################################### +def _replace(string): + """Replace the variables the manual page uses.""" + + result = string + + for variable, value in _VAR_MAP.items(): + result = result.replace(variable, value) + + if "{[" in result: + raise ToolError("unreplaced variable(s) in: %s" % string) + + return result + + +#################################################################################################################################### +def _split_size(string, size): + """Split a paragraph into lines no longer than a size, breaking at spaces. + + A word longer than the size is left on a line of its own rather than broken, so a path or a url stays readable.""" + + result = [] + base = 0 + match_last = None + + while True: + match = string.find(" ", base if match_last is None else match_last) + + if match == -1: + # Whatever is left is the last line, less a break when what is left is still too long + if match_last is not None and len(string) - base - 1 >= size: + result.append(string[base : match_last - 1]) + base = match_last + + result.append(string[base:]) + + return result + + # A word that would take the line past the size starts the next line, breaking at the space before it when there is one + if match - base >= size: + if match_last is not None: + match = match_last - 1 + + result.append(string[base:match]) + base = match + 1 + match_last = None + else: + match_last = match + 1 + + +#################################################################################################################################### +def _render_text(text, indent, indent_first): + """Wrap text to the console width, indenting every line but optionally the first. + + The first line is not indented where the text follows a name on the same line, e.g. a command and its summary.""" + + result = "" + + for line in text.split("\n"): + if result != "": + result += "\n" + + for part_idx, part in enumerate(_split_size(line, _CONSOLE_WIDTH - indent)): + if part_idx != 0 or indent_first: + if part_idx != 0: + result += "\n" + + if len(part) > 0: + result += " " * indent + + result += part + + return result + + +#################################################################################################################################### +def _command_list(bld_cfg, bld_hlp): + """The commands to list, i.e. every command a reader can run.""" + + cmd_cfg_map = {cmd_cfg.name: cmd_cfg for cmd_cfg in bld_cfg.cmd_list} + + return [cmd_hlp for cmd_hlp in bld_hlp.cmd_list if not cmd_cfg_map[cmd_hlp.name].internal] + + +#################################################################################################################################### +def _section_list(bld_cfg, bld_hlp, cmd_hlp_list): + """The options to list, grouped by section, and the width the widest of them needs. + + An option a command documents itself is listed under that command, since what it means there is what was documented.""" + + # The help is checked against the declarations when it is parsed, but only that everything declared has help. Help for something + # that is not declared is a leftover, and it is reported here rather than left to fail as a missing key. + opt_cfg_map = {opt_cfg.name: opt_cfg for opt_cfg in bld_cfg.opt_list} + + section_map = {} + width_max = 0 + + def add(section, opt_hlp): + section_map.setdefault(section, []).append(opt_hlp) + + def opt_cfg_find(name, where): + result = opt_cfg_map.get(name) + check(result is not None, "help %s documents option '%s', which is not declared" % (where, name)) + + return result + + for cmd_hlp in cmd_hlp_list: + for opt_hlp in cmd_hlp.opt_list or []: + opt_cfg = opt_cfg_find(opt_hlp.name, "for command '%s'" % cmd_hlp.name) + opt_cmd_cfg = _find(opt_cfg.cmd_list, cmd_hlp.name) + + check( + opt_cmd_cfg is not None, + "help for command '%s' documents option '%s', which the command does not take" % (cmd_hlp.name, opt_hlp.name), + ) + + # Skip an option the command marks internal, since a reader has no use for it + if opt_cmd_cfg.internal: + continue + + width_max = max(width_max, len(opt_hlp.name)) + add(cmd_hlp.name[:1].upper() + cmd_hlp.name[1:], opt_hlp) + + for opt_hlp in bld_hlp.opt_list: + if opt_cfg_find(opt_hlp.name, "in the option list").internal: + continue + + width_max = max(width_max, len(opt_hlp.name)) + + section = _SECTION_DEFAULT if opt_hlp.section is None else opt_hlp.section[:1].upper() + opt_hlp.section[1:] + + add(section, opt_hlp) + section_map[section].sort(key=lambda opt_hlp: opt_hlp.name) + + return sorted(section_map.items()), width_max + + +#################################################################################################################################### +def reference_man_render(index, bld_cfg, bld_hlp): + """Render the manual page.""" + + # What the project is, which is taken from the documentation index rather than said again here + subtitle = _replace(xml_node_attribute(index, "subtitle", True)) + description = _render_text(_replace(xml_node_content(xml_node_child(index, "description", True))), 2, True) + + result = "NAME\n %s - %s\n\nSYNOPSIS\n %s [options] [command]\n\nDESCRIPTION\n%s\n" % ( + _NAME, + subtitle, + _BIN, + description, + ) + + cmd_hlp_list = _command_list(bld_cfg, bld_hlp) + width_max = max(len(cmd_hlp.name) for cmd_hlp in cmd_hlp_list) + + result += "\nCOMMANDS\n" + + for cmd_hlp in cmd_hlp_list: + summary = _render_text(bld_hlp_render_xml(cmd_hlp.summary), width_max + 4, False) + + result += " %s %*s%s\n" % (cmd_hlp.name, width_max - len(cmd_hlp.name), "", summary) + + section_list, width_max = _section_list(bld_cfg, bld_hlp, cmd_hlp_list) + + result += "\nOPTIONS\n" + + for section_idx, (section, opt_hlp_list) in enumerate(section_list): + if section_idx != 0: + result += "\n" + + result += " %s Options:\n" % section + + for opt_hlp in opt_hlp_list: + summary = _render_text(bld_hlp_render_xml(opt_hlp.summary), width_max + 8, False) + + result += " --%s %*s%s\n" % (opt_hlp.name, width_max - len(opt_hlp.name), "", summary) + + return result + _FILES + _EXAMPLES + _SEE_ALSO diff --git a/doc/lib/command/build/pre.py b/doc/lib/command/build/pre.py new file mode 100644 index 000000000..14a79455d --- /dev/null +++ b/doc/lib/command/build/pre.py @@ -0,0 +1,111 @@ +"""Build Preprocessor. + +Prepares a document for rendering by resolving everything that is written once and used many times: the variables a document declares, +the blocks it defines and repeats with different values, the descriptions it takes from the help, and the conditions that decide which +parts of it apply at all. + +Everything here happens before a renderer sees the document, so a renderer only ever deals with content that is meant to be there.""" + +#################################################################################################################################### +from command.build.eval import eval_expression +from command.build.var_store import VarStore +from common.error import ToolError, check +from common.xml import ( + xml_node_attribute, + xml_node_attribute_remove, + xml_node_child_list, + xml_node_child_replace, + xml_node_content, + xml_node_dup, +) + + +#################################################################################################################################### +def _find(item_list, name): + """Find an entry in a list by name, or None when it is not there.""" + + for item in item_list or []: + if item.name == name: + return item + + return None + + +#################################################################################################################################### +def _description(node, parent, item_list, key, what): + """Replace a node with the description the help holds for what it names.""" + + item = _find(item_list, key) + check(item is not None, "%s '%s' has no help to describe it" % (what, key)) + + xml_node_child_replace(parent, node, item.description) + + +#################################################################################################################################### +def _block_replace(node, parent, block, bld_hlp, block_map, var_store): + """Replace a node with a copy of the block it names, with the values it gives the block filled in.""" + + # Values for this use of the block, which are variables that apply to the copy only + var_block_store = VarStore() + + for var_replace in xml_node_child_list(node, "block-variable-replace"): + var_block_store.add(xml_node_attribute(var_replace, "key", True), xml_node_content(var_replace)) + + copy = xml_node_dup(block) + var_block_store.replace_node(copy) + + # A block may use another block, so what it holds is preprocessed the same as anything else + _pre_recurse(copy, bld_hlp, block_map, var_store) + + xml_node_child_replace(parent, node, copy) + + +#################################################################################################################################### +def _pre_recurse(node, bld_hlp, block_map, var_store): + """Preprocess everything under a node.""" + + # The children are taken first because handling one can remove it + for child in list(node): + # A node with a condition stays only when the condition holds, and the condition itself is not rendered + if_expr = xml_node_attribute(child, "if") + + if if_expr is not None: + if not eval_expression(var_store.replace_str(if_expr)): + node.remove(child) + + continue + + xml_node_attribute_remove(child, "if") + + if child.tag == "option-description": + _description(child, node, bld_hlp.opt_list, xml_node_attribute(child, "key", True), "option") + elif child.tag == "cmd-description": + _description(child, node, bld_hlp.cmd_list, xml_node_attribute(child, "key", True), "command") + elif child.tag == "variable": + var_store.add(xml_node_attribute(child, "key", True), var_store.replace_str(xml_node_content(child))) + elif child.tag == "block-define": + id = xml_node_attribute(child, "id", True) + + if id in block_map: + raise ToolError("block '%s' is already defined" % id) + + block_map[id] = xml_node_dup(child) + node.remove(child) + elif child.tag == "block": + id = xml_node_attribute(child, "id", True) + + if id not in block_map: + raise ToolError("block '%s' does not exist" % id) + + _block_replace(child, node, block_map[id], bld_hlp, block_map, var_store) + else: + _pre_recurse(child, bld_hlp, block_map, var_store) + + +#################################################################################################################################### +def build_pre(document, bld_hlp, var_store): + """Preprocess a document and return it.""" + + _pre_recurse(document.root, bld_hlp, {}, var_store) + + return document diff --git a/doc/lib/command/build/reference.py b/doc/lib/command/build/reference.py new file mode 100644 index 000000000..7721c4f67 --- /dev/null +++ b/doc/lib/command/build/reference.py @@ -0,0 +1,300 @@ +"""Build Command and Configuration Reference. + +Renders the reference documents from the same declarations the binary is generated from, so what is documented is what the code does +rather than a second description of it that has to be kept in step. + +The configuration reference documents every option that can be set in the configuration file, grouped by the section it belongs to. The +command reference documents every command and, under each, the options that command takes, grouped by what they are about. An option +therefore appears in both, described once and rendered twice.""" + +#################################################################################################################################### +from common.xml import xml_document_new, xml_node_add, xml_node_attribute_set, xml_node_child_add, xml_node_content_add +from config.parse import DEFAULT_TYPE_DYNAMIC, OPT_TYPE_BOOLEAN, SECTION_COMMAND_LINE + +# Role a user gives an option, as opposed to a role a command starts for itself +_CMD_ROLE_MAIN = "main" + +# Section an option is documented under when it has none of its own +_SECTION_DEFAULT = "general" + +# Sections that mean the same thing in the command reference as in the configuration reference. Anything else is about the command +# itself and is grouped under one heading there. +_SECTION_COMMAND_LIST = ("general", "log", "maintainer", "repository", "stanza") + +_SECTION_COMMAND = "command" + + +#################################################################################################################################### +def _find(item_list, name): + """Find an entry in a list by name, or None when it is not there.""" + + for item in item_list or []: + if item.name == name: + return item + + return None + + +#################################################################################################################################### +def _default_block(block_list, opt_cfg, default): + """Describe the default of an option, which may be a value per value of the option it depends on.""" + + # A dynamic default is worked out when the command runs, so what it depends on is described rather than a value + if opt_cfg.default_type == DEFAULT_TYPE_DYNAMIC: + block_list.append("default: [path of executed pgbackrest binary]") + elif default is not None: + if default.value is not None: + # A boolean is written the way it is given on the command line rather than the way it is declared + if opt_cfg.type == OPT_TYPE_BOOLEAN: + block_list.append("default: %s" % ("y" if default.value == "true" else "n")) + else: + block_list.append("default: %s" % default.value) + else: + block_list.append("default (depending on %s):" % opt_cfg.depend.option.name) + + for map in default.map_list: + block_list.append(" %s - %s" % (map.map, map.value)) + + block_list.append("") + + +#################################################################################################################################### +def _allow_range_block(block_list, opt_cfg): + """Describe the range an option allows, which may be a range per value of the option it depends on.""" + + if opt_cfg.allow_range.map_list is not None: + block_list.append("allow range (depending on %s):" % opt_cfg.depend.option.name) + + for map in opt_cfg.allow_range.map_list: + block_list.append(" %s - [%s, %s]" % (map.map, map.min, map.max)) + + block_list.append("") + else: + block_list.append("allowed: [%s, %s]" % (opt_cfg.allow_range.min, opt_cfg.allow_range.max)) + + +#################################################################################################################################### +def _example_block(block_list, opt_cfg, opt_hlp, command): + """Show how an option is given, which is on the command line for a command and in the configuration file otherwise.""" + + # An option in a group is indexed, and the example shows the first index since that is what a reader will start with + option = opt_cfg.name + + if opt_cfg.group is not None: + option = "%s1%s" % (opt_cfg.group, opt_cfg.name[len(opt_cfg.group) :]) + + output = "example: " + + for example_idx, example in enumerate(opt_hlp.example_list): + if command: + if example_idx != 0: + output += " " + + output += "--" + + # A boolean is given by naming it, so turning it off is naming the negation of it + if opt_cfg.type == OPT_TYPE_BOOLEAN and example == "n": + output += "no-" + # In the configuration file each example is a line of its own, since that is how it would be written + elif example_idx != 0: + block_list.append(output) + output = "example: " + + output += option + + if not command or opt_cfg.type != OPT_TYPE_BOOLEAN: + output += "=%s" % example + + block_list.append(output) + + +#################################################################################################################################### +def _option_render(xml_section, opt_cmd_cfg, opt_cfg, opt_hlp): + """Render one option, either as a command takes it or as the configuration file sets it.""" + + xml_option = xml_node_add(xml_section, "section") + xml_title = xml_node_add(xml_option, "title") + + xml_node_attribute_set(xml_option, "id", "option-%s" % opt_hlp.name) + xml_node_content_add(xml_title, "%s Option (" % opt_hlp.title) + xml_node_content_add(xml_node_add(xml_title, "id"), "--%s" % opt_hlp.name) + xml_node_content_add(xml_title, ")") + xml_node_child_add(xml_node_add(xml_option, "p"), opt_hlp.summary) + + if opt_cfg.beta: + xml_node_content_add(xml_node_add(xml_option, "p"), "FOR BETA TESTING ONLY. DO NOT USE IN PRODUCTION.") + + xml_node_child_add(xml_option, opt_hlp.description) + + # What a reader needs to use the option, which is rendered as one block below the description + block_list = [] + default = ( + opt_cmd_cfg.default_value if opt_cmd_cfg is not None and opt_cmd_cfg.default_value is not None else opt_cfg.default_value + ) + + _default_block(block_list, opt_cfg, default) + + if opt_cfg.allow_range is not None: + _allow_range_block(block_list, opt_cfg) + + if opt_hlp.example_list is not None: + _example_block(block_list, opt_cfg, opt_hlp, opt_cmd_cfg is not None) + + if len(block_list) > 0: + xml_node_content_add(xml_node_add(xml_option, "code-block"), "\n".join(block_list).strip()) + + # Names the option can still be given by, less any that is the option name itself since that is not a deprecated name + if opt_cfg.deprecate_list is not None: + deprecate = "" + + for entry in opt_cfg.deprecate_list: + if entry.name != opt_cfg.name: + deprecate += "%s %s" % ("," if deprecate != "" else "", entry.name) + + if deprecate != "": + xml_node_content_add( + xml_node_add(xml_option, "p"), + "Deprecated Name%s:%s" % ("s" if len(opt_cfg.deprecate_list) > 1 else "", deprecate), + ) + + +#################################################################################################################################### +def _document_new(title, description, introduction): + """Build a reference document, which begins with what it is and an introduction to it.""" + + result = xml_document_new("doc", dtd_name="doc", dtd_file="doc.dtd") + + xml_node_attribute_set(result.root, "title", "{[project]}") + xml_node_attribute_set(result.root, "subtitle", title) + xml_node_attribute_set(result.root, "toc", "y") + + xml_node_content_add(xml_node_add(result.root, "description"), description) + + xml_intro = xml_node_add(result.root, "section") + + xml_node_attribute_set(xml_intro, "id", "introduction") + xml_node_content_add(xml_node_add(xml_intro, "title"), "Introduction") + xml_node_child_add(xml_intro, introduction) + + return result + + +#################################################################################################################################### +def reference_configuration_render(bld_cfg, bld_hlp): + """Render the configuration reference, i.e. every option that can be set in the configuration file.""" + + result = _document_new(bld_hlp.opt_title, bld_hlp.opt_description, bld_hlp.opt_introduction) + + for section in bld_hlp.sct_list: + xml_section = xml_node_add(result.root, "section") + + xml_node_attribute_set(xml_section, "id", "section-%s" % section.id) + xml_node_content_add(xml_node_add(xml_section, "title"), "%s Options" % section.name) + xml_node_child_add(xml_section, section.introduction) + + for opt_hlp in bld_hlp.opt_list: + opt_cfg = _find(bld_cfg.opt_list, opt_hlp.name) + + # Skip an option documented under another section, one that cannot be set in the configuration file at all, and one that + # is internal since a reader has no use for it + if (opt_hlp.section or _SECTION_DEFAULT) != section.id: + continue + + if opt_cfg.section == SECTION_COMMAND_LINE or opt_cfg.internal: + continue + + _option_render(xml_section, None, opt_cfg, opt_hlp) + + return result + + +#################################################################################################################################### +def _command_section(section): + """The section an option is grouped under in the command reference. + + A section that means the same thing everywhere keeps its name. Anything else is about the command itself, so it is grouped under one + heading rather than a heading per command.""" + + if section is None: + return _SECTION_DEFAULT + + return section if section in _SECTION_COMMAND_LIST else _SECTION_COMMAND + + +#################################################################################################################################### +class _CommandOption: + """An option as one command takes it, i.e. the option, what the command says about it, and the help to render.""" + + def __init__(self, opt_cfg, opt_cmd_cfg, opt_hlp, section): + self.opt_cfg = opt_cfg + self.opt_cmd_cfg = opt_cmd_cfg + self.opt_hlp = opt_hlp + self.section = section + + +#################################################################################################################################### +def _command_option_list(cmd_cfg, cmd_hlp, bld_cfg, bld_hlp): + """The options a command takes, with the help to render for each.""" + + result = [] + + for opt_cfg in bld_cfg.opt_list: + # Skip an option a reader has no use for or should not be told about + if opt_cfg.internal or opt_cfg.secure: + continue + + # Skip an option this command does not take, or takes only for a role the user does not start + opt_cmd_cfg = _find(opt_cfg.cmd_list, cmd_cfg.name) + + if opt_cmd_cfg is None or opt_cmd_cfg.internal or _CMD_ROLE_MAIN not in opt_cmd_cfg.role_list: + continue + + # Help a command documents differently wins over the help the option has of its own + opt_hlp = _find(cmd_hlp.opt_list, opt_cfg.name) + section = _SECTION_COMMAND if opt_hlp is not None else None + + if opt_hlp is None: + opt_hlp = _find(bld_hlp.opt_list, opt_cfg.name) + section = opt_hlp.section + + result.append(_CommandOption(opt_cfg, opt_cmd_cfg, opt_hlp, _command_section(section))) + + return result + + +#################################################################################################################################### +def reference_command_render(bld_cfg, bld_hlp): + """Render the command reference, i.e. every command and the options it takes.""" + + result = _document_new(bld_hlp.cmd_title, bld_hlp.cmd_description, bld_hlp.cmd_introduction) + + for cmd_hlp in bld_hlp.cmd_list: + cmd_cfg = _find(bld_cfg.cmd_list, cmd_hlp.name) + + # Skip a command a reader has no use for + if cmd_cfg.internal: + continue + + xml_section = xml_node_add(result.root, "section") + xml_title = xml_node_add(xml_section, "title") + + xml_node_attribute_set(xml_section, "id", "command-%s" % cmd_hlp.name) + xml_node_content_add(xml_title, "%s Command (" % cmd_hlp.title) + xml_node_content_add(xml_node_add(xml_title, "id"), cmd_hlp.name) + xml_node_content_add(xml_title, ")") + xml_node_child_add(xml_section, cmd_hlp.description) + + opt_list = _command_option_list(cmd_cfg, cmd_hlp, bld_cfg, bld_hlp) + + for section in sorted({opt.section for opt in opt_list}): + xml_category = xml_node_add(xml_section, "section") + + xml_node_attribute_set(xml_category, "id", "category-%s" % section) + xml_node_attribute_set(xml_category, "toc", "n") + xml_node_content_add(xml_node_add(xml_category, "title"), "%s Options" % (section[:1].upper() + section[1:])) + + for opt in opt_list: + if opt.section == section: + _option_render(xml_category, opt.opt_cmd_cfg, opt.opt_cfg, opt.opt_hlp) + + return result diff --git a/doc/lib/command/build/var_store.py b/doc/lib/command/build/var_store.py new file mode 100644 index 000000000..ebf025125 --- /dev/null +++ b/doc/lib/command/build/var_store.py @@ -0,0 +1,57 @@ +"""Variable Store. + +Holds the variables a document refers to as {[name]} and replaces them wherever they appear. A value may itself refer to a variable, +so replacement repeats until nothing more is replaced rather than passing over the text once.""" + +#################################################################################################################################### +from common.xml import xml_node_attribute, xml_node_attribute_set + + +#################################################################################################################################### +class VarStore: + """A set of variables and their values.""" + + def __init__(self): + self.var_map = {} + + ################################################################################################################################ + def add(self, variable, value): + """Add a variable, keeping the value it already has when it has one. + + The first value wins because the command line is loaded before the document, so what the caller asked for is not overridden by + what the document declares.""" + + self.var_map.setdefault("{[%s]}" % variable, value) + + ################################################################################################################################ + def replace_str(self, string): + """Replace every variable in a string.""" + + result = string + replace = True + + while replace: + replace = False + + for variable, value in self.var_map.items(): + if variable in result: + result = result.replace(variable, value) + replace = True + + return result + + ################################################################################################################################ + def replace_node(self, node): + """Replace every variable in the attributes and the text of a node and everything under it.""" + + for name in list(node.attrib): + xml_node_attribute_set(node, name, self.replace_str(xml_node_attribute(node, name))) + + if node.text is not None: + node.text = self.replace_str(node.text) + + for child in node: + self.replace_node(child) + + if child.tail is not None: + child.tail = self.replace_str(child.tail) diff --git a/doc/manifest.xml b/doc/manifest.xml index 811ac9c92..882675b6a 100644 --- a/doc/manifest.xml +++ b/doc/manifest.xml @@ -75,7 +75,7 @@ - + diff --git a/doc/src/build/config/config.yaml b/doc/src/build/config/config.yaml deleted file mode 100644 index fdd3626de..000000000 --- a/doc/src/build/config/config.yaml +++ /dev/null @@ -1,136 +0,0 @@ -#################################################################################################################################### -# Configuration Definition -#################################################################################################################################### - -#################################################################################################################################### -# Commands -#################################################################################################################################### -command: - help: - log-level-default: DEBUG - parameter-allowed: true - - build: - parameter-allowed: true - - noop: - internal: true - - version: - log-level-default: DEBUG - -#################################################################################################################################### -# Option groups that are not used but must be present for modules to compile -#################################################################################################################################### -optionGroup: - pg: {} - repo: {} - -#################################################################################################################################### -# Options -#################################################################################################################################### -option: - # General options - #--------------------------------------------------------------------------------------------------------------------------------- - buffer-size: - type: size - internal: true - default: 64KiB - allow-list: - - 16KiB - - 32KiB - - 64KiB - - 128KiB - - 256KiB - - 512KiB - - 1MiB - - help: - type: boolean - default: false - command: - help: {} - command-role: - main: {} - - neutral-umask: - type: boolean - internal: true - default: true - - output: - type: string-id - sequence: true - command: - version: - default: text - allow-list: - - num - - text - command-role: - main: {} - - repo-path: - type: string - default: pgbackrest - command: - build: {} - - version: - type: boolean - default: false - command: - help: {} - command-role: - main: {} - - # Logging options - #--------------------------------------------------------------------------------------------------------------------------------- - log-level: - type: string-id - sequence: true - default: info - allow-list: - - off - - error - - warn - - info - - detail - - debug - - trace - - log-timestamp: - type: boolean - default: true - negate: true - command: log-level - - # Build options - #--------------------------------------------------------------------------------------------------------------------------------- - var: - type: hash - required: false - command: - build: {} - - # Options that are not used but must be present for modules to compile. All must have a default or not be required. - #--------------------------------------------------------------------------------------------------------------------------------- - beta: {type: boolean, default: false, command: {noop: {}}} - cmd: {type: string, default-type: dynamic, default: bin, command: {noop: {}}} - compress-level-network: {type: string, required: false, command: {noop: {}}} - config: { - type: string, internal: true, default-type: literal, default: CFGOPTDEF_CONFIG_PATH "/" PROJECT_CONFIG_FILE, negate: true} - config-path: {type: string, required: false, command: {noop: {}}} - config-include-path: { - type: string, default-type: literal, default: CFGOPTDEF_CONFIG_PATH "/" PROJECT_CONFIG_INCLUDE_PATH, command: {noop: {}}} - job-retry: {type: time, default: 0, deprecate: {job-retry-old: {}}, command: {noop: {}}} - job-retry-interval: {type: integer, default: 0, command: {noop: {}}} - lock: {type: list, required: false, command: {noop: {}}} - log-level-file: {type: string, required: false, command: {noop: {}}} - log-level-stderr: {type: string, required: false, command: {noop: {}}} - pg: {type: string, required: false, command: {noop: {}}} - pg-path: {type: string, required: false, command: {noop: {}}} - repo-type: {type: string, required: false, command: {noop: {}}} - repo: {type: string, required: false, command: {noop: {}}} - spool-path: {type: string, required: false, command: {noop: {}}} - stanza: {type: string, required: false, command: {noop: {}}} diff --git a/doc/src/build/help/help.xml b/doc/src/build/help/help.xml deleted file mode 100644 index 6da62f583..000000000 --- a/doc/src/build/help/help.xml +++ /dev/null @@ -1,174 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Get help. - - -

Three levels of help are provided. If no command is specified then general help will be displayed. If a command is specified (e.g. doc-pgbackrest help build) then a full description of the command will be displayed along with a list of valid options. If an option is specified in addition to a command (e.g. doc-pgbackrest help build repo-path) then a full description of the option as it applies to the command will be displayed.

- - - - - - - - - - -

- - - Build documentation. - - -

Build the specified documentation module.

- - - - - - - - - Get version. - - -

Displays installed version.

- - - - - - - - - diff --git a/doc/src/command/build/build.c b/doc/src/command/build/build.c deleted file mode 100644 index 1fedeba1c..000000000 --- a/doc/src/command/build/build.c +++ /dev/null @@ -1,63 +0,0 @@ -/*********************************************************************************************************************************** -Build Command -***********************************************************************************************************************************/ -#include - -#include "command/build/build.h" -#include "command/build/man.h" -#include "command/build/pre.h" -#include "command/build/reference.h" -#include "common/debug.h" -#include "common/log.h" -#include "storage/posix/storage.h" -#include "version.h" - -/**********************************************************************************************************************************/ -void -cmdBuild(const String *const pathRepo, const KeyValue *const varKv) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STRING, pathRepo); - FUNCTION_LOG_PARAM(KEY_VALUE, varKv); - FUNCTION_LOG_END(); - - MEM_CONTEXT_TEMP_BEGIN() - { - VarStore *const varStore = varStoreNew(); - Storage *const storageRepo = storagePosixNewP(pathRepo, .write = true); - const BldCfg bldCfg = bldCfgParse(storageRepo); - const BldHlp bldHlp = bldHlpParse(storageRepo, bldCfg, true); - XmlNode *const index = xmlDocumentRoot( - xmlDocumentNewBuf(storageGetP(storageNewReadP(storageRepo, STRDEF("doc/xml/index.xml"))))); - XmlDocument *const userGuide = xmlDocumentNewBuf( - storageGetP(storageNewReadP(storageRepo, STRDEF("doc/xml/user-guide.xml")))); - - // Load variables from command-line - if (varKv != NULL) - { - const VariantList *const varList = kvKeyList(varKv); - - for (unsigned int varIdx = 0; varIdx < varLstSize(varList); varIdx++) - { - const Variant *const key = varLstGet(varList, varIdx); - varStoreAdd(varStore, varStr(key), varStr(kvGet(varKv, key))); - } - } - - storagePutP( - storageNewWriteP(storageRepo, STRDEF("doc/output/xml/command.xml")), - xmlDocumentBuf(referenceCommandRender(&bldCfg, &bldHlp))); - storagePutP( - storageNewWriteP(storageRepo, STRDEF("doc/output/xml/configuration.xml")), - xmlDocumentBuf(referenceConfigurationRender(&bldCfg, &bldHlp))); - storagePutP( - storageNewWriteP(storageRepo, STRDEF("doc/output/xml/user-guide.xml")), - xmlDocumentBuf(buildPre(userGuide, &bldCfg, &bldHlp, varStore))); - storagePutP( - storageNewWriteP(storageRepo, STRDEF("doc/output/man/" PROJECT_BIN ".1.txt")), - BUFSTR(referenceManRender(index, &bldCfg, &bldHlp))); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN_VOID(); -} diff --git a/doc/src/command/build/build.h b/doc/src/command/build/build.h deleted file mode 100644 index 9e9ec59e0..000000000 --- a/doc/src/command/build/build.h +++ /dev/null @@ -1,14 +0,0 @@ -/*********************************************************************************************************************************** -Build Documentation -***********************************************************************************************************************************/ -#ifndef DOC_COMMAND_BUILD_H -#define DOC_COMMAND_BUILD_H - -#include "common/type/keyValue.h" - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -void cmdBuild(const String *pathRepo, const KeyValue *varKv); - -#endif diff --git a/doc/src/command/build/eval.c b/doc/src/command/build/eval.c deleted file mode 100644 index a7d51a69c..000000000 --- a/doc/src/command/build/eval.c +++ /dev/null @@ -1,217 +0,0 @@ -/*********************************************************************************************************************************** -Expression Evaluation -***********************************************************************************************************************************/ -#include - -#include -#include - -#include "command/build/eval.h" -#include "common/debug.h" - -/*********************************************************************************************************************************** -Numeric comparison operator types -***********************************************************************************************************************************/ -typedef enum -{ - evalOpTypeGt, - evalOpTypeGte, - evalOpTypeLt, - evalOpTypeLte, -} EvalOpType; - -/*********************************************************************************************************************************** -Find an operator at the top level (not inside parentheses) -***********************************************************************************************************************************/ -static const char * -evalFindOp(const char *const expr, const char *const op) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(STRINGZ, expr); - FUNCTION_TEST_PARAM(STRINGZ, op); - FUNCTION_TEST_END(); - - int depth = 0; - const char *result = NULL; - - for (const char *pos = expr; *pos != '\0'; pos++) - { - if (*pos == '(') - depth++; - else if (*pos == ')') - depth--; - else if (depth == 0 && strncmp(pos, op, strlen(op)) == 0) - { - result = pos; - break; - } - } - - FUNCTION_TEST_RETURN_TYPE_P(const char, result); -} - -/*********************************************************************************************************************************** -Evaluate a single expression (no &&/|| operators) -***********************************************************************************************************************************/ -static bool -evalOne(const String *const expression) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(STRING, expression); - FUNCTION_TEST_END(); - - bool result; - - MEM_CONTEXT_TEMP_BEGIN() - { - const String *const trimmed = strTrim(strDup(expression)); - const char *const expr = strZ(trimmed); - - // Parenthesized expression - if (expr[0] == '(') - { - CHECK_FMT( - FormatError, strSize(trimmed) >= 2 && expr[strSize(trimmed) - 1] == ')', - "unmatched parenthesis in '%s'", strZ(expression)); - - result = eval(strNewZN(expr + 1, strSize(trimmed) - 2)); - } - // Negation - else if (expr[0] == '!') - { - result = !evalOne(strNewZ(expr + 1)); - } - // String comparison with eq/ne - else if (strstr(expr, " eq ") != NULL || strstr(expr, " ne ") != NULL) - { - const char *eqOp = strstr(expr, " eq "); - const char *neOp = strstr(expr, " ne "); - const bool isEq = eqOp != NULL; - const char *const op = isEq ? eqOp : neOp; - const size_t opLen = 4; - - // Extract left operand (strip surrounding quotes) - const String *left = strTrim(strNewZN(expr, (size_t)(op - expr))); - - CHECK_FMT( - FormatError, strSize(left) >= 2 && strZ(left)[0] == '\'' && strZ(left)[strSize(left) - 1] == '\'', - "expected quoted string on left side of '%s': '%s'", isEq ? "eq" : "ne", strZ(expression)); - - left = strNewZN(strZ(left) + 1, strSize(left) - 2); - - // Extract right operand (strip surrounding quotes) - const String *right = strTrim(strNewZ(op + opLen)); - - CHECK_FMT( - FormatError, strSize(right) >= 2 && strZ(right)[0] == '\'' && strZ(right)[strSize(right) - 1] == '\'', - "expected quoted string on right side of '%s': '%s'", isEq ? "eq" : "ne", strZ(expression)); - - right = strNewZN(strZ(right) + 1, strSize(right) - 2); - - result = isEq ? strEq(left, right) : !strEq(left, right); - } - // Numeric comparison - else if (strstr(expr, " >= ") != NULL || strstr(expr, " <= ") != NULL || strstr(expr, " > ") != NULL || - strstr(expr, " < ") != NULL) - { - const char *op; - size_t opLen; - EvalOpType opType; - - if ((op = strstr(expr, " >= ")) != NULL) - { - opLen = 4; - opType = evalOpTypeGte; - } - else if ((op = strstr(expr, " <= ")) != NULL) - { - opLen = 4; - opType = evalOpTypeLte; - } - else if ((op = strstr(expr, " > ")) != NULL) - { - opLen = 3; - opType = evalOpTypeGt; - } - else - { - op = strstr(expr, " < "); - opLen = 3; - opType = evalOpTypeLt; - } - - const int left = atoi(expr); - const int right = atoi(op + opLen); - - switch (opType) - { - case evalOpTypeGt: - result = left > right; - break; - - case evalOpTypeGte: - result = left >= right; - break; - - case evalOpTypeLt: - result = left < right; - break; - - case evalOpTypeLte: - result = left <= right; - break; - } - } - else - CHECK_FMT(FormatError, false, "unable to evaluate '%s'", strZ(expression)); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_TEST_RETURN(BOOL, result); -} - -/**********************************************************************************************************************************/ -FN_EXTERN bool -eval(const String *const expression) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(STRING, expression); - FUNCTION_TEST_END(); - - ASSERT(expression != NULL); - CHECK_FMT(AssertError, strstr(strZ(expression), "{[") == NULL, "unreplaced var in expression '%s'", strZ(expression)); - - bool result; - - MEM_CONTEXT_TEMP_BEGIN() - { - const char *const expr = strZ(expression); - - // Split on || (lowest precedence, left to right, skip parens) - const char *const orOp = evalFindOp(expr, " || "); - - if (orOp != NULL) - { - result = eval(strTrim(strNewZN(expr, (size_t)(orOp - expr)))) || - eval(strTrim(strNewZ(orOp + 4))); - } - else - { - // Split on && (next precedence, left to right, skip parens) - const char *const andOp = evalFindOp(expr, " && "); - - if (andOp != NULL) - { - result = eval(strTrim(strNewZN(expr, (size_t)(andOp - expr)))) && - eval(strTrim(strNewZ(andOp + 4))); - } - else - { - result = evalOne(expression); - } - } - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_TEST_RETURN(BOOL, result); -} diff --git a/doc/src/command/build/eval.h b/doc/src/command/build/eval.h deleted file mode 100644 index b416b7fe4..000000000 --- a/doc/src/command/build/eval.h +++ /dev/null @@ -1,25 +0,0 @@ -/*********************************************************************************************************************************** -Expression Evaluation - -Simple expression evaluator that handles the subset of expressions used in doc XML if attributes. - -Supports: - - Negation: !expr - - Parentheses: (expr) - - String comparison: 'str1' eq 'str2', 'str1' ne 'str2' - - Numeric comparison: num > num, num >= num, num < num, num <= num - - Logical and: expr && expr - - Logical or: expr || expr -***********************************************************************************************************************************/ -#ifndef DOC_COMMAND_EVAL_H -#define DOC_COMMAND_EVAL_H - -#include "common/type/string.h" - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Evaluate an expression and return the boolean result -FN_EXTERN bool eval(const String *expression); - -#endif diff --git a/doc/src/command/build/man.c b/doc/src/command/build/man.c deleted file mode 100644 index 98b9091d1..000000000 --- a/doc/src/command/build/man.c +++ /dev/null @@ -1,299 +0,0 @@ -/*********************************************************************************************************************************** -Build Manual Page Reference -***********************************************************************************************************************************/ -#include - -#include "build/common/string.h" -#include "build/help/render.h" -#include "command/build/man.h" -#include "command/help/help.h" -#include "common/debug.h" -#include "common/log.h" -#include "version.h" - -/*********************************************************************************************************************************** -Define the console width - use a fixed width of 80 since this should be safe on virtually all consoles -***********************************************************************************************************************************/ -#define CONSOLE_WIDTH 80 - -/*********************************************************************************************************************************** -Basic variable replacement. This only handles a few cases but will error in case of unknown variables. -***********************************************************************************************************************************/ -static String * -referenceManReplace(String *const string) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(STRING, string); - FUNCTION_TEST_END(); - - String *const result = strCat(strNew(), string); - - strReplace(result, STRDEF("{[postgres]}"), STRDEF("PostgreSQL")); - strReplace(result, STRDEF("{[project]}"), STRDEF("pgBackRest")); - - if (strstr(strZ(result), "{[") != NULL) - THROW_FMT(AssertError, "unreplaced variable(s) in: %s", strZ(string)); - - FUNCTION_TEST_RETURN(STRING, result); -} - -/**********************************************************************************************************************************/ -typedef struct ReferenceManSectionData -{ - const String *name; // Section name - List *optList; // Option list -} ReferenceManSectionData; - -// Helper to calculate max width -static size_t -referenceManMaxWidth(const size_t widthMax, const String *const string) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(SIZE, widthMax); - FUNCTION_TEST_PARAM(STRING, string); - FUNCTION_TEST_END(); - - if (strSize(string) > widthMax) - FUNCTION_TEST_RETURN(SIZE, strSize(string)); - - FUNCTION_TEST_RETURN(SIZE, widthMax); -} - -String * -referenceManRender(const XmlNode *const indexRoot, const BldCfg *const bldCfg, const BldHlp *const bldHlp) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(XML_NODE, indexRoot); - FUNCTION_LOG_PARAM_P(VOID, bldCfg); - FUNCTION_LOG_PARAM_P(VOID, bldHlp); - FUNCTION_LOG_END(); - - String *const result = strNew(); - - MEM_CONTEXT_TEMP_BEGIN() - { - // Output header - const String *const subtitle = referenceManReplace(xmlNodeAttribute(indexRoot, STRDEF("subtitle"))); - const String *const description = helpRenderText( - referenceManReplace(xmlNodeContent(xmlNodeChild(indexRoot, STRDEF("description"), true))), false, false, 2, true, - CONSOLE_WIDTH); - - strCatFmt( - result, - "NAME\n" - " " PROJECT_NAME " - %s\n\n" - "SYNOPSIS\n" - " " PROJECT_BIN " [options] [command]\n\n" - "DESCRIPTION\n" - "%s\n", - strZ(subtitle), strZ(description)); - - // Output commands - // ------------------------------------------------------------------------------------------------------------------------- - // Determine max command width - size_t commandWidthMax = 0; - - for (unsigned int cmdIdx = 0; cmdIdx < lstSize(bldHlp->cmdList); cmdIdx++) - { - const BldHlpCommand *const cmdHlp = lstGet(bldHlp->cmdList, cmdIdx); - const BldCfgCommand *const cmdCfg = lstFind(bldCfg->cmdList, &cmdHlp->name); - ASSERT(cmdCfg != NULL); - - // Skip internal commands - if (cmdCfg->internal) - continue; - - // Update max command width - commandWidthMax = referenceManMaxWidth(commandWidthMax, cmdHlp->name); - } - - // Output commands - strCatZ(result, "\nCOMMANDS\n"); - - for (unsigned int cmdIdx = 0; cmdIdx < lstSize(bldHlp->cmdList); cmdIdx++) - { - const BldHlpCommand *const cmdHlp = lstGet(bldHlp->cmdList, cmdIdx); - const BldCfgCommand *const cmdCfg = lstFind(bldCfg->cmdList, &cmdHlp->name); - ASSERT(cmdCfg != NULL); - - // Skip internal commands - if (cmdCfg->internal) - continue; - - strCatFmt( - result, " %s %*s%s\n", strZ(cmdHlp->name), - (int)(commandWidthMax - strSize(cmdHlp->name)), "", - strZ(helpRenderText(bldHlpRenderXml(cmdHlp->summary), false, false, commandWidthMax + 4, false, CONSOLE_WIDTH))); - } - - // Output options - // ------------------------------------------------------------------------------------------------------------------------- - // Determine max option width and build list of sections/options - List *const sectionList = lstNewP(sizeof(ReferenceManSectionData), .comparator = lstComparatorStr); - size_t optionWidthMax = 0; - - for (unsigned int cmdIdx = 0; cmdIdx < lstSize(bldHlp->cmdList); cmdIdx++) - { - const BldHlpCommand *const cmdHlp = lstGet(bldHlp->cmdList, cmdIdx); - const BldCfgCommand *const cmdCfg = lstFind(bldCfg->cmdList, &cmdHlp->name); - ASSERT(cmdCfg != NULL); - - // Skip internal commands - if (cmdCfg->internal) - continue; - - if (cmdHlp->optList != NULL) - { - // Get section name - const String *const sectionName = strFirstUpper(strDup(cmdHlp->name)); - - for (unsigned int optIdx = 0; optIdx < lstSize(cmdHlp->optList); optIdx++) - { - const BldHlpOption *const optHlp = lstGet(cmdHlp->optList, optIdx); - const BldCfgOption *const optCfg = lstFind(bldCfg->optList, &optHlp->name); - ASSERT(optCfg != NULL); - - // Skip internal options - const BldCfgOptionCommand *const optCmdCfg = lstFind(optCfg->cmdList, &cmdHlp->name); - ASSERT(optCmdCfg != NULL); - - if (optCmdCfg->internal) - continue; - - // Update max option width - optionWidthMax = referenceManMaxWidth(optionWidthMax, optHlp->name); - - // Add/get section - ReferenceManSectionData *section = lstFind(sectionList, §ionName); - - if (section == NULL) - { - MEM_CONTEXT_OBJ_BEGIN(sectionList) - { - const ReferenceManSectionData sectionNew = - { - .name = strDup(sectionName), - .optList = lstNewP(sizeof(BldHlpOption), .comparator = lstComparatorStr), - }; - - section = lstAdd(sectionList, §ionNew); - } - MEM_CONTEXT_OBJ_END(); - } - - // Add option - lstAdd(section->optList, optHlp); - } - } - } - - for (unsigned int optIdx = 0; optIdx < lstSize(bldHlp->optList); optIdx++) - { - const BldHlpOption *const optHlp = lstGet(bldHlp->optList, optIdx); - const BldCfgOption *const optCfg = lstFind(bldCfg->optList, &optHlp->name); - ASSERT(optCfg != NULL); - - // Skip if option is internal - if (optCfg->internal) - continue; - - // Update max option width - optionWidthMax = referenceManMaxWidth(optionWidthMax, optHlp->name); - - // Get section name - const String *const sectionName = optHlp->section == NULL ? STRDEF("General") : strFirstUpper(strDup(optHlp->section)); - - // Add/get section - ReferenceManSectionData *section = lstFind(sectionList, §ionName); - - if (section == NULL) - { - MEM_CONTEXT_OBJ_BEGIN(sectionList) - { - const ReferenceManSectionData sectionNew = - { - .name = strDup(sectionName), - .optList = lstNewP(sizeof(BldHlpOption), .comparator = lstComparatorStr), - }; - - section = lstAdd(sectionList, §ionNew); - } - MEM_CONTEXT_OBJ_END(); - } - - // Add option - lstAdd(section->optList, optHlp); - lstSort(section->optList, sortOrderAsc); - } - - lstSort(sectionList, sortOrderAsc); - - // Output options - strCatZ(result, "\nOPTIONS\n"); - - for (unsigned int sectionIdx = 0; sectionIdx < lstSize(sectionList); sectionIdx++) - { - const ReferenceManSectionData *const section = lstGet(sectionList, sectionIdx); - - if (sectionIdx != 0) - strCatZ(result, "\n"); - - strCatFmt(result, " %s Options:\n", strZ(section->name)); - - for (unsigned int optIdx = 0; optIdx < lstSize(section->optList); optIdx++) - { - const BldHlpOption *const optHlp = lstGet(section->optList, optIdx); - - strCatFmt( - result, " --%s %*s%s\n", strZ(optHlp->name), - (int)(optionWidthMax - strSize(optHlp->name)), "", - strZ(helpRenderText(bldHlpRenderXml(optHlp->summary), false, false, optionWidthMax + 8, false, CONSOLE_WIDTH))); - } - } - - // Output files - // ------------------------------------------------------------------------------------------------------------------------- - strCatZ( - result, - "\nFILES\n" - " " CFGOPTDEF_CONFIG_PATH "/" PROJECT_CONFIG_FILE "\n" - " /var/lib/pgbackrest\n" - " /var/log/pgbackrest\n" - " /var/spool/pgbackrest\n" - " /tmp/pgbackrest\n"); - - // Output examples - // ------------------------------------------------------------------------------------------------------------------------- - strCatZ( - result, - "\nEXAMPLES\n" - " * Create a backup of the PostgreSQL `main` cluster:\n" - "\n" - " $ pgbackrest --stanza=main backup\n" - "\n" - " The `main` cluster should be configured in `" CFGOPTDEF_CONFIG_PATH "/" PROJECT_CONFIG_FILE "`\n" - "\n" - " * Show all available backups:\n" - "\n" - " $ pgbackrest info\n" - "\n" - " * Show all available backups for a specific cluster:\n" - "\n" - " $ pgbackrest --stanza=main info\n" - "\n" - " * Show backup specific options:\n" - "\n" - " $ pgbackrest help backup\n"); - - // Output see also - // ------------------------------------------------------------------------------------------------------------------------- - strCatZ( - result, - "\nSEE ALSO\n" - " /usr/share/doc/pgbackrest-doc/html/index.html\n" - " https://pgbackrest.org\n"); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN(STRING, result); -} diff --git a/doc/src/command/build/man.h b/doc/src/command/build/man.h deleted file mode 100644 index fc52bdf27..000000000 --- a/doc/src/command/build/man.h +++ /dev/null @@ -1,17 +0,0 @@ -/*********************************************************************************************************************************** -Build Manual Page Reference -***********************************************************************************************************************************/ -#ifndef DOC_COMMAND_BUILD_MAN_H -#define DOC_COMMAND_BUILD_MAN_H - -#include "build/config/parse.h" -#include "build/help/parse.h" -#include "common/type/xml.h" - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Build manual page reference -String *referenceManRender(const XmlNode *indexRoot, const BldCfg *bldCfg, const BldHlp *bldHlp); - -#endif diff --git a/doc/src/command/build/pre.c b/doc/src/command/build/pre.c deleted file mode 100644 index b72ba371c..000000000 --- a/doc/src/command/build/pre.c +++ /dev/null @@ -1,178 +0,0 @@ -/*********************************************************************************************************************************** -Build Preprocessor -***********************************************************************************************************************************/ -#include - -#include "build/common/string.h" -#include "command/build/eval.h" -#include "command/build/pre.h" -#include "common/debug.h" -#include "common/log.h" -#include "common/type/list.h" - -/*********************************************************************************************************************************** -Block define -***********************************************************************************************************************************/ -typedef struct BuildBlockDefine -{ - const String *id; // Block id - const XmlNode *block; // Block xml -} BuildBlockDefine; - -/**********************************************************************************************************************************/ -static void -buildPreRecurse( - XmlNode *const xml, const BldCfg *bldCfg, const BldHlp *bldHlp, List *blockDefineList, VarStore *const varStore) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(XML_DOCUMENT, xml); - FUNCTION_LOG_PARAM_P(VOID, bldCfg); - FUNCTION_LOG_PARAM_P(VOID, bldHlp); - FUNCTION_LOG_PARAM(LIST, blockDefineList); - FUNCTION_LOG_PARAM(VAR_STORE, varStore); - FUNCTION_LOG_END(); - - ASSERT(xml != NULL); - ASSERT(bldCfg != NULL); - ASSERT(bldHlp != NULL); - ASSERT(blockDefineList != NULL); - ASSERT(varStore != NULL); - - MEM_CONTEXT_TEMP_BEGIN() - { - // Iterate all nodes - const XmlNodeList *const nodeList = xmlNodeChildListAll(xml, false); - - for (unsigned int nodeIdx = 0; nodeIdx < xmlNodeLstSize(nodeList); nodeIdx++) - { - XmlNode *const node = xmlNodeLstGet(nodeList, nodeIdx); - - // Evaluate if expressions - const String *const ifExpr = xmlNodeAttribute(node, STRDEF("if")); - - if (ifExpr != NULL) - { - // If expression is true then remove if attribute - if (eval(varStoreReplaceStr(varStore, ifExpr))) - { - xmlNodeAttributeRemove(node, STRDEF("if")); - } - // Else remove node - else - { - xmlNodeRemove(node); - continue; - } - } - - // Replace option description - if (strEqZ(xmlNodeName(node), "option-description")) - { - const String *const key = xmlNodeAttribute(node, STRDEF("key")); - ASSERT(key != NULL); - const BldHlpOption *const hlpOpt = lstFind(bldHlp->optList, &key); - ASSERT(hlpOpt != NULL); - - xmlNodeChildReplace(node, hlpOpt->description); - } - // Replace command description - else if (strEqZ(xmlNodeName(node), "cmd-description")) - { - const String *const key = xmlNodeAttribute(node, STRDEF("key")); - ASSERT(key != NULL); - const BldHlpCommand *const hlpCmd = lstFind(bldHlp->cmdList, &key); - ASSERT(hlpCmd != NULL); - - xmlNodeChildReplace(node, hlpCmd->description); - } - // Store variable - else if (strEqZ(xmlNodeName(node), "variable")) - { - const String *const variable = xmlNodeAttribute(node, STRDEF("key")); - CHECK(FormatError, variable != NULL, "key is missing"); - const String *const value = varStoreReplaceStr(varStore, xmlNodeContent(node)); - CHECK(FormatError, value != NULL, "value is missing"); - - varStoreAdd(varStore, variable, value); - } - // Block define - else if (strEqZ(xmlNodeName(node), "block-define")) - { - const String *const id = xmlNodeAttribute(node, STRDEF("id")); - CHECK(FormatError, id != NULL, "block id is missing"); - CHECK_FMT(FormatError, lstFind(blockDefineList, &id) == NULL, "block '%s' is already defined", strZ(id)); - - MEM_CONTEXT_BEGIN(lstMemContext(blockDefineList)) - { - const BuildBlockDefine blockDefine = - { - .id = id, - .block = xmlNodeDup(node), - }; - - lstAdd(blockDefineList, &blockDefine); - } - MEM_CONTEXT_END(); - - xmlNodeRemove(node); - } - // Block replace - else if (strEqZ(xmlNodeName(node), "block")) - { - const String *const id = xmlNodeAttribute(node, STRDEF("id")); - CHECK(FormatError, id != NULL, "block id is missing"); - const BuildBlockDefine *const blockDefine = lstFind(blockDefineList, &id); - CHECK_FMT(FormatError, blockDefine != NULL, "block '%s' does not exist", strZ(id)); - - // Get variable replacements - VarStore *const varBlockStore = varStoreNew(); - const XmlNodeList *const varReplaceList = xmlNodeChildList(node, STRDEF("block-variable-replace")); - - for (unsigned int varReplaceIdx = 0; varReplaceIdx < xmlNodeLstSize(varReplaceList); varReplaceIdx++) - { - const XmlNode *const varReplace = xmlNodeLstGet(varReplaceList, varReplaceIdx); - const String *const variable = xmlNodeAttribute(varReplace, STRDEF("key")); - CHECK(FormatError, variable != NULL, "key is missing"); - const String *const value = xmlNodeContent(varReplace); - CHECK(FormatError, value != NULL, "value is missing"); - - varStoreAdd(varBlockStore, variable, value); - } - - // Copy node, do variable replacement, recurse, and replace in document - XmlNode *const block = xmlNodeDup(blockDefine->block); - varStoreReplaceNode(varBlockStore, block); - buildPreRecurse(block, bldCfg, bldHlp, blockDefineList, varStore); - xmlNodeChildReplace(node, block); - } - // Else recurse - else - buildPreRecurse(node, bldCfg, bldHlp, blockDefineList, varStore); - } - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN_VOID(); -} - -/**********************************************************************************************************************************/ -XmlDocument * -buildPre(XmlDocument *const xml, const BldCfg *bldCfg, const BldHlp *bldHlp, VarStore *const varStore) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(XML_DOCUMENT, xml); - FUNCTION_LOG_PARAM_P(VOID, bldCfg); - FUNCTION_LOG_PARAM_P(VOID, bldHlp); - FUNCTION_LOG_PARAM(VAR_STORE, varStore); - FUNCTION_LOG_END(); - - MEM_CONTEXT_TEMP_BEGIN() - { - List *const blockDefineList = lstNewP(sizeof(BuildBlockDefine), .comparator = lstComparatorStr); - - buildPreRecurse(xmlDocumentRoot(xml), bldCfg, bldHlp, blockDefineList, varStore); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN(XML_DOCUMENT, xml); -} diff --git a/doc/src/command/build/pre.h b/doc/src/command/build/pre.h deleted file mode 100644 index d8c831236..000000000 --- a/doc/src/command/build/pre.h +++ /dev/null @@ -1,18 +0,0 @@ -/*********************************************************************************************************************************** -Build Preprocessor -***********************************************************************************************************************************/ -#ifndef DOC_COMMAND_BUILD_PRE_H -#define DOC_COMMAND_BUILD_PRE_H - -#include "build/config/parse.h" -#include "build/help/parse.h" -#include "command/build/varStore.h" -#include "common/type/xml.h" - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Build command reference -XmlDocument *buildPre(XmlDocument *xml, const BldCfg *bldCfg, const BldHlp *bldHlp, VarStore *varStore); - -#endif diff --git a/doc/src/command/build/reference.c b/doc/src/command/build/reference.c deleted file mode 100644 index 9c7f3e323..000000000 --- a/doc/src/command/build/reference.c +++ /dev/null @@ -1,379 +0,0 @@ -/*********************************************************************************************************************************** -Build Command and Configuration Reference -***********************************************************************************************************************************/ -#include - -#include "build/config/parse.h" -#include "build/help/parse.h" -#include "command/build/build.h" -#include "command/build/reference.h" -#include "common/debug.h" -#include "common/log.h" -#include "storage/posix/storage.h" -#include "storage/storage.h" -#include "version.h" - -/*********************************************************************************************************************************** -Build option -***********************************************************************************************************************************/ -static void -referenceOptionRender( - XmlNode *const xmlSection, const BldCfgOptionCommand *const optCmdCfg, const BldCfgOption *const optCfg, - const BldHlpOption *const optHlp) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(XML_NODE, xmlSection); - FUNCTION_LOG_PARAM_P(VOID, optCmdCfg); - FUNCTION_LOG_PARAM_P(VOID, optCfg); - FUNCTION_LOG_PARAM_P(VOID, optHlp); - FUNCTION_LOG_END(); - - MEM_CONTEXT_TEMP_BEGIN() - { - XmlNode *const xmlOption = xmlNodeAdd(xmlSection, STRDEF("section")); - XmlNode *const xmlOptionTitle = xmlNodeAdd(xmlOption, STRDEF("title")); - - xmlNodeAttributeSet(xmlOption, STRDEF("id"), strNewFmt("option-%s", strZ(optHlp->name))); - xmlNodeContentSet(xmlOptionTitle, strNewFmt("%s Option (", strZ(optHlp->title))); - xmlNodeContentSet(xmlNodeAdd(xmlOptionTitle, STRDEF("id")), strNewFmt("--%s", strZ(optHlp->name))); - xmlNodeContentSet(xmlOptionTitle, STRDEF(")")); - xmlNodeChildAdd(xmlNodeAdd(xmlOption, STRDEF("p")), optHlp->summary); - - if (optCfg->beta) - xmlNodeContentSet(xmlNodeAdd(xmlOption, STRDEF("p")), STRDEF("FOR BETA TESTING ONLY. DO NOT USE IN PRODUCTION.")); - - xmlNodeChildAdd(xmlOption, optHlp->description); - - // Add default value or default map - StringList *const blockList = strLstNew(); - const BldCfgOptionDefault *const defaultValue = - optCmdCfg != NULL && optCmdCfg->defaultValue != NULL ? optCmdCfg->defaultValue : optCfg->defaultValue; - - if (optCfg->defaultType == defaultTypeDynamic) - { - ASSERT(strEqZ(defaultValue->value, "bin")); - - strLstAddZ(blockList, "default: [path of executed pgbackrest binary]"); - } - else if (defaultValue != NULL) - { - if (defaultValue->value != NULL) - { - if (strEq(optCfg->type, OPT_TYPE_BOOLEAN_STR)) - strLstAddFmt(blockList, "default: %s", strEqZ(defaultValue->value, "true") ? "y" : "n"); - else - strLstAddFmt(blockList, "default: %s", strZ(defaultValue->value)); - } - else - { - strLstAddFmt(blockList, "default (depending on %s):", strZ(optCfg->depend->option->name)); - - for (unsigned int mapIdx = 0; mapIdx < lstSize(defaultValue->mapList); mapIdx++) - { - const BldCfgOptionDefaultMap *const map = lstGet(defaultValue->mapList, mapIdx); - strLstAddFmt(blockList, " %s - %s", strZ(map->map), strZ(map->value)); - } - - strLstAddZ(blockList, ""); - } - } - - // Add allow range - if (optCfg->allowRange != NULL) - { - if (optCfg->allowRange->mapList != NULL) - { - strLstAddFmt(blockList, "allow range (depending on %s):", strZ(optCfg->depend->option->name)); - - for (unsigned int mapIdx = 0; mapIdx < lstSize(optCfg->allowRange->mapList); mapIdx++) - { - const BldCfgOptionAllowRangeMap *const map = lstGet(optCfg->allowRange->mapList, mapIdx); - strLstAddFmt(blockList, " %s - [%s, %s]", strZ(map->map), strZ(map->min), strZ(map->max)); - } - - strLstAddZ(blockList, ""); - } - else - strLstAddFmt(blockList, "allowed: [%s, %s]", strZ(optCfg->allowRange->min), strZ(optCfg->allowRange->max)); - } - - // Add examples - if (optHlp->exampleList != NULL) - { - String *output = strCatZ(strNew(), "example: "); - const String *const option = - optCfg->group != NULL ? - strNewFmt("%s1%s", strZ(optCfg->group), strZ(strSub(optCfg->name, strSize(optCfg->group)))) : optCfg->name; - - for (unsigned int exampleIdx = 0; exampleIdx < strLstSize(optHlp->exampleList); exampleIdx++) - { - const String *const example = strLstGet(optHlp->exampleList, exampleIdx); - - // If Command line example - if (optCmdCfg != NULL) - { - if (exampleIdx != 0) - strCatZ(output, " "); - - strCatZ(output, "--"); - - if (strEq(optCfg->type, OPT_TYPE_BOOLEAN_STR) && strEqZ(example, "n")) - strCatZ(output, "no-"); - } - // Else configuration file example - else if (exampleIdx != 0) - { - strLstAdd(blockList, output); - output = strCatZ(strNew(), "example: "); - } - - strCat(output, option); - - if (optCmdCfg == NULL || !strEq(optCfg->type, OPT_TYPE_BOOLEAN_STR)) - strCatFmt(output, "=%s", strZ(example)); - } - - strLstAdd(blockList, output); - } - - if (!strLstEmpty(blockList)) - xmlNodeContentSet(xmlNodeAdd(xmlOption, STRDEF("code-block")), strTrim(strLstJoin(blockList, "\n"))); - - // Add deprecated names - if (optCfg->deprecateList != NULL) - { - String *const deprecateStr = strNew(); - - for (unsigned int deprecateIdx = 0; deprecateIdx < lstSize(optCfg->deprecateList); deprecateIdx++) - { - const BldCfgOptionDeprecate *const deprecate = lstGet(optCfg->deprecateList, deprecateIdx); - - if (!strEq(deprecate->name, optCfg->name)) - strCatFmt(deprecateStr, "%s %s", !strEmpty(deprecateStr) ? "," : "", strZ(deprecate->name)); - } - - if (!strEmpty(deprecateStr)) - { - xmlNodeContentSet( - xmlNodeAdd(xmlOption, STRDEF("p")), - strNewFmt("Deprecated Name%s:%s", lstSize(optCfg->deprecateList) > 1 ? "s" : "", strZ(deprecateStr))); - } - } - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN_VOID(); -} - -/**********************************************************************************************************************************/ -XmlDocument * -referenceConfigurationRender(const BldCfg *const bldCfg, const BldHlp *const bldHlp) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM_P(VOID, bldCfg); - FUNCTION_LOG_PARAM_P(VOID, bldHlp); - FUNCTION_LOG_END(); - - XmlDocument *const result = xmlDocumentNewP(STRDEF("doc"), .dtdName = STRDEF("doc"), .dtdFile = STRDEF("doc.dtd")); - - MEM_CONTEXT_TEMP_BEGIN() - { - XmlNode *const xmlRoot = xmlDocumentRoot(result); - - // Set attributes in root node - xmlNodeAttributeSet(xmlRoot, STRDEF("title"), STRDEF("{[project]}")); - xmlNodeAttributeSet(xmlRoot, STRDEF("subtitle"), bldHlp->optTitle); - xmlNodeAttributeSet(xmlRoot, STRDEF("toc"), STRDEF("y")); - - // Set description - xmlNodeContentSet(xmlNodeAdd(xmlRoot, STRDEF("description")), bldHlp->optDescription); - - // Set introduction - XmlNode *const xmlIntro = xmlNodeAdd(xmlRoot, STRDEF("section")); - - xmlNodeAttributeSet(xmlIntro, STRDEF("id"), STRDEF("introduction")); - xmlNodeContentSet(xmlNodeAdd(xmlIntro, STRDEF("title")), STRDEF("Introduction")); - xmlNodeChildAdd(xmlIntro, bldHlp->optIntroduction); - - for (unsigned int sctIdx = 0; sctIdx < lstSize(bldHlp->sctList); sctIdx++) - { - const BldHlpSection *const section = lstGet(bldHlp->sctList, sctIdx); - XmlNode *const xmlSection = xmlNodeAdd(xmlRoot, STRDEF("section")); - XmlNode *const xmlSectionTitle = xmlNodeAdd(xmlSection, STRDEF("title")); - - xmlNodeAttributeSet(xmlSection, STRDEF("id"), strNewFmt("section-%s", strZ(section->id))); - xmlNodeContentSet(xmlSectionTitle, strNewFmt("%s Options", strZ(section->name))); - xmlNodeChildAdd(xmlSection, section->introduction); - - for (unsigned int optIdx = 0; optIdx < lstSize(bldHlp->optList); optIdx++) - { - const BldHlpOption *const optHlp = lstGet(bldHlp->optList, optIdx); - const BldCfgOption *const optCfg = lstFind(bldCfg->optList, &optHlp->name); - ASSERT(optCfg != NULL); - - // Skip if option does not belong in this section - if (!strEq(optHlp->section == NULL ? STRDEF("general") : optHlp->section, section->id)) - continue; - - // Skip if option is command-line only - if (strEq(optCfg->section, SECTION_COMMAND_LINE_STR)) - continue; - - // Skip if option is internal - if (optCfg->internal) - continue; - - referenceOptionRender(xmlSection, NULL, optCfg, optHlp); - } - } - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN(XML_DOCUMENT, result); -} - -/**********************************************************************************************************************************/ -// Helper to remap section names for the command reference -static String * -referenceCommandSection(const String *const section) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(STRING, section); - FUNCTION_TEST_END(); - - if (section == NULL) - FUNCTION_TEST_RETURN(STRING, strNewZ("general")); - - if (!strEqZ(section, "general") && !strEqZ(section, "log") && !strEqZ(section, "maintainer") && - !strEqZ(section, "repository") && !strEqZ(section, "stanza")) - { - FUNCTION_TEST_RETURN(STRING, strNewZ("command")); - } - - FUNCTION_TEST_RETURN(STRING, strDup(section)); -} - -XmlDocument * -referenceCommandRender(const BldCfg *const bldCfg, const BldHlp *const bldHlp) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM_P(VOID, bldCfg); - FUNCTION_LOG_PARAM_P(VOID, bldHlp); - FUNCTION_LOG_END(); - - XmlDocument *const result = xmlDocumentNewP(STRDEF("doc"), .dtdName = STRDEF("doc"), .dtdFile = STRDEF("doc.dtd")); - - MEM_CONTEXT_TEMP_BEGIN() - { - XmlNode *const xmlRoot = xmlDocumentRoot(result); - - // Set attributes in root node - xmlNodeAttributeSet(xmlRoot, STRDEF("title"), STRDEF("{[project]}")); - xmlNodeAttributeSet(xmlRoot, STRDEF("subtitle"), bldHlp->cmdTitle); - xmlNodeAttributeSet(xmlRoot, STRDEF("toc"), STRDEF("y")); - - // Set description - xmlNodeContentSet(xmlNodeAdd(xmlRoot, STRDEF("description")), bldHlp->cmdDescription); - - // Set introduction - XmlNode *const xmlIntro = xmlNodeAdd(xmlRoot, STRDEF("section")); - - xmlNodeAttributeSet(xmlIntro, STRDEF("id"), STRDEF("introduction")); - xmlNodeContentSet(xmlNodeAdd(xmlIntro, STRDEF("title")), STRDEF("Introduction")); - xmlNodeChildAdd(xmlIntro, bldHlp->cmdIntroduction); - - for (unsigned int cmdIdx = 0; cmdIdx < lstSize(bldHlp->cmdList); cmdIdx++) - { - const BldHlpCommand *const cmdHlp = lstGet(bldHlp->cmdList, cmdIdx); - const BldCfgCommand *const cmdCfg = lstFind(bldCfg->cmdList, &cmdHlp->name); - ASSERT(cmdCfg != NULL); - - // Skip internal commands - if (cmdCfg->internal) - continue; - - XmlNode *const xmlSection = xmlNodeAdd(xmlRoot, STRDEF("section")); - XmlNode *const xmlSectionTitle = xmlNodeAdd(xmlSection, STRDEF("title")); - - xmlNodeAttributeSet(xmlSection, STRDEF("id"), strNewFmt("command-%s", strZ(cmdHlp->name))); - xmlNodeContentSet(xmlSectionTitle, strNewFmt("%s Command (", strZ(cmdHlp->title))); - xmlNodeContentSet(xmlNodeAdd(xmlSectionTitle, STRDEF("id")), cmdHlp->name); - xmlNodeContentSet(xmlSectionTitle, STRDEF(")")); - xmlNodeChildAdd(xmlSection, cmdHlp->description); - - // Build option list for command - StringList *const sctList = strLstNew(); - List *const optCfgList = lstNewP(sizeof(BldCfgOption)); - List *const optHlpList = lstNewP(sizeof(BldHlpOption)); - - for (unsigned int optIdx = 0; optIdx < lstSize(bldCfg->optList); optIdx++) - { - const BldCfgOption *const optCfg = lstGet(bldCfg->optList, optIdx); - - // Skip internal options - if (optCfg->internal || optCfg->secure) - continue; - - // Skip options that are not valid, internal, or do not have the main role for this command - const BldCfgOptionCommand *const optCmdCfg = lstFind(optCfg->cmdList, &cmdCfg->name); - - if (optCmdCfg == NULL || optCmdCfg->internal || !strLstExists(optCmdCfg->roleList, STRDEF("main"))) - continue; - - // Get help from command list or general list - BldHlpOption optHlp = {0}; - - if (cmdHlp->optList != NULL && lstFind(cmdHlp->optList, &optCfg->name) != NULL) - { - optHlp = *((const BldHlpOption *)lstFind(cmdHlp->optList, &optCfg->name)); - optHlp.section = strNewZ("command"); - } - else - { - ASSERT(lstFind(bldHlp->optList, &optCfg->name) != NULL); - optHlp = *((const BldHlpOption *)lstFind(bldHlp->optList, &optCfg->name)); - } - - // Remap section name - optHlp.section = referenceCommandSection(optHlp.section); - - // Add sections - strLstAddIfMissing(sctList, optHlp.section); - - // Add options - lstAdd(optCfgList, optCfg); - lstAdd(optHlpList, &optHlp); - } - - strLstSort(sctList, sortOrderAsc); - - for (unsigned int sctIdx = 0; sctIdx < strLstSize(sctList); sctIdx++) - { - const String *const section = strLstGet(sctList, sctIdx); - XmlNode *const xmlCommandCategory = xmlNodeAdd(xmlSection, STRDEF("section")); - XmlNode *const xmlCommandCategoryTitle = xmlNodeAdd(xmlCommandCategory, STRDEF("title")); - - xmlNodeAttributeSet(xmlCommandCategory, STRDEF("id"), strNewFmt("category-%s", strZ(section))); - xmlNodeAttributeSet(xmlCommandCategory, STRDEF("toc"), STRDEF("n")); - xmlNodeContentSet(xmlCommandCategoryTitle, strNewFmt("%s Options", strZ(strFirstUpper(strDup(section))))); - - for (unsigned int optIdx = 0; optIdx < lstSize(optHlpList); optIdx++) - { - const BldHlpOption *const optHlp = lstGet(optHlpList, optIdx); - const BldCfgOption *const optCfg = lstGet(optCfgList, optIdx); - ASSERT(optCfg != NULL); - const BldCfgOptionCommand *const optCmdCfg = lstFind(optCfg->cmdList, &cmdCfg->name); - ASSERT(optCmdCfg != NULL); - - if (!strEq(optHlp->section, section)) - continue; - - referenceOptionRender(xmlCommandCategory, optCmdCfg, optCfg, optHlp); - } - } - } - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN(XML_DOCUMENT, result); -} diff --git a/doc/src/command/build/reference.h b/doc/src/command/build/reference.h deleted file mode 100644 index d1e5bb16e..000000000 --- a/doc/src/command/build/reference.h +++ /dev/null @@ -1,20 +0,0 @@ -/*********************************************************************************************************************************** -Build Command and Configuration Reference -***********************************************************************************************************************************/ -#ifndef DOC_COMMAND_BUILD_REFERENCE_H -#define DOC_COMMAND_BUILD_REFERENCE_H - -#include "build/config/parse.h" -#include "build/help/parse.h" -#include "common/type/xml.h" - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Build command reference -XmlDocument *referenceCommandRender(const BldCfg *bldCfg, const BldHlp *bldHlp); - -// Build configuration reference -XmlDocument *referenceConfigurationRender(const BldCfg *bldCfg, const BldHlp *bldHlp); - -#endif diff --git a/doc/src/command/build/varStore.c b/doc/src/command/build/varStore.c deleted file mode 100644 index ad2c8978c..000000000 --- a/doc/src/command/build/varStore.c +++ /dev/null @@ -1,152 +0,0 @@ -/*********************************************************************************************************************************** -Variable Store -***********************************************************************************************************************************/ -#include - -#include - -#include "build/common/string.h" -#include "build/common/xml.h" -#include "command/build/varStore.h" -#include "common/debug.h" -#include "common/memContext.h" - -/*********************************************************************************************************************************** -Object type -***********************************************************************************************************************************/ -struct VarStore -{ - List *varList; // Variable list -}; - -typedef struct VarStoreData -{ - const String *variable; // Variable - const String *value; // Value -} VarStoreData; - -/**********************************************************************************************************************************/ -FN_EXTERN VarStore * -varStoreNew(void) -{ - FUNCTION_TEST_VOID(); - - OBJ_NEW_BEGIN(VarStore, .childQty = MEM_CONTEXT_QTY_MAX) - { - *this = (VarStore) - { - .varList = lstNewP(sizeof(VarStoreData), .comparator = lstComparatorStr), - }; - } - OBJ_NEW_END(); - - FUNCTION_TEST_RETURN(VAR_STORE, this); -} - -/**********************************************************************************************************************************/ -FN_EXTERN void -varStoreAdd(VarStore *const this, const String *const variable, const String *const value) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(VAR_STORE, this); - FUNCTION_TEST_PARAM(STRING, variable); - FUNCTION_TEST_PARAM(STRING, value); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - ASSERT(variable != NULL); - ASSERT(value != NULL); - - MEM_CONTEXT_OBJ_BEGIN(this) - { - const struct VarStoreData varStoreData = - { - .variable = strNewFmt("{[%s]}", strZ(variable)), - .value = strDup(value), - }; - - if (!lstFind(this->varList, &varStoreData.variable)) - { - lstAdd(this->varList, &varStoreData); - lstSort(this->varList, sortOrderAsc); - } - } - MEM_CONTEXT_OBJ_END(); - - FUNCTION_TEST_RETURN_VOID(); -} - -/**********************************************************************************************************************************/ -FN_EXTERN String * -varStoreReplaceStr(VarStore *const this, const String *const string) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(VAR_STORE, this); - FUNCTION_TEST_PARAM(STRING, string); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - ASSERT(string != NULL); - - String *const result = strCat(strNew(), string); - unsigned int replaceTotal; - - do - { - replaceTotal = 0; - - for (unsigned int varIdx = 0; varIdx < lstSize(this->varList); varIdx++) - { - const VarStoreData *const varData = lstGet(this->varList, varIdx); - replaceTotal += strReplace(result, varData->variable, varData->value); - } - } - while (replaceTotal > 0); - - FUNCTION_TEST_RETURN(STRING, result); -} - -/**********************************************************************************************************************************/ -FN_EXTERN void -varStoreReplaceNode(VarStore *const this, XmlNode *const node) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(VAR_STORE, this); - FUNCTION_TEST_PARAM(XML_NODE, node); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - ASSERT(node != NULL); - - MEM_CONTEXT_TEMP_BEGIN() - { - // Replace variables in attributes of the current node - const StringList *const attrList = xmlNodeAttributeList(node); - - for (unsigned int attrIdx = 0; attrIdx < strLstSize(attrList); attrIdx++) - { - const String *const attrName = strLstGet(attrList, attrIdx); - xmlNodeAttributeSet(node, attrName, varStoreReplaceStr(this, xmlNodeAttribute(node, attrName))); - } - - // Iterate child nodes and replace variables in text nodes - const XmlNodeList *const childList = xmlNodeChildListAll(node, true); - - for (unsigned int childIdx = 0; childIdx < xmlNodeLstSize(childList); childIdx++) - { - XmlNode *const child = xmlNodeLstGet(childList, childIdx); - - // Replace variables in text node content - if (strEqZ(xmlNodeName(child), "text")) - { - xmlNodeTextSet(child, varStoreReplaceStr(this, xmlNodeContent(child))); - } - // Recurse into element nodes - else - varStoreReplaceNode(this, child); - } - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_TEST_RETURN_VOID(); -} diff --git a/doc/src/command/build/varStore.h b/doc/src/command/build/varStore.h deleted file mode 100644 index 1bc54e9e6..000000000 --- a/doc/src/command/build/varStore.h +++ /dev/null @@ -1,39 +0,0 @@ -/*********************************************************************************************************************************** -Variable Store -***********************************************************************************************************************************/ -#ifndef DOC_COMMAND_VARSTORE_H -#define DOC_COMMAND_VARSTORE_H - -/*********************************************************************************************************************************** -Object type -***********************************************************************************************************************************/ -typedef struct VarStore VarStore; - -#include "common/type/xml.h" - -/*********************************************************************************************************************************** -Constructor -***********************************************************************************************************************************/ -FN_EXTERN VarStore *varStoreNew(void); - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Add variable -FN_EXTERN void varStoreAdd(VarStore *this, const String *variable, const String *value); - -// Replace variables in a node -FN_EXTERN void varStoreReplaceNode(VarStore *this, XmlNode *node); - -// Replace variables in a string -FN_EXTERN String *varStoreReplaceStr(VarStore *this, const String *string); - -/*********************************************************************************************************************************** -Macros for function logging -***********************************************************************************************************************************/ -#define FUNCTION_LOG_VAR_STORE_TYPE \ - VarStore * -#define FUNCTION_LOG_VAR_STORE_FORMAT(value, buffer, bufferSize) \ - objNameToLog(value, "VarStore", buffer, bufferSize) - -#endif diff --git a/doc/src/command/help/meson.build b/doc/src/command/help/meson.build deleted file mode 100644 index e38c2afea..000000000 --- a/doc/src/command/help/meson.build +++ /dev/null @@ -1,17 +0,0 @@ -#################################################################################################################################### -# Generate help -#################################################################################################################################### -doc_help_auto_c_inc = custom_target( - 'doc_help.auto.c.inc', - output: 'help.auto.c.inc', - depend_files: [ - '../../build/config/config.yaml', - '../../build/help/help.xml', - ], - command: [ - build_code, - 'help', - '@CURRENT_SOURCE_DIR@/../../..', - '@BUILD_ROOT@/doc', - ], -) diff --git a/doc/src/config/load.c b/doc/src/config/load.c deleted file mode 100644 index 62001f142..000000000 --- a/doc/src/config/load.c +++ /dev/null @@ -1,120 +0,0 @@ -/*********************************************************************************************************************************** -Configuration Load -***********************************************************************************************************************************/ -#include - -#include - -#include "command/command.h" -#include "common/debug.h" -#include "common/io/io.h" -#include "common/log.h" -#include "config/config.intern.h" -#include "config/load.h" -#include "storage/posix/storage.h" - -/*********************************************************************************************************************************** -Load log settings -***********************************************************************************************************************************/ -static void -cfgLoadLogSetting(void) -{ - FUNCTION_LOG_VOID(logLevelTrace); - - // Initialize logging - LogLevel logLevelConsole = logLevelOff; - bool logTimestamp = true; - - if (cfgOptionValid(cfgOptLogLevel)) - logLevelConsole = logLevelEnum(cfgOptionSeq(cfgOptLogLevel)); - - if (cfgOptionValid(cfgOptLogTimestamp)) - logTimestamp = cfgOptionBool(cfgOptLogTimestamp); - - logInit(logLevelConsole, logLevelOff, logLevelOff, logTimestamp, 0, 1, false); - - FUNCTION_LOG_RETURN_VOID(); -} - -/*********************************************************************************************************************************** -Update options that have complex rules -***********************************************************************************************************************************/ -static void -cfgLoadUpdateOption(void) -{ - FUNCTION_LOG_VOID(logLevelTrace); - - // Get current working dir - char currentWorkDir[1024]; - THROW_ON_SYS_ERROR(getcwd(currentWorkDir, sizeof(currentWorkDir)) == NULL, FormatError, "unable to get cwd"); - - // If repo-path is relative then make it absolute - if (cfgOptionValid(cfgOptRepoPath)) - { - const String *const repoPath = cfgOptionStr(cfgOptRepoPath); - - if (!strBeginsWithZ(repoPath, "/")) - { - cfgOptionSet( - cfgOptRepoPath, cfgOptionSource(cfgOptRepoPath), VARSTR(strNewFmt("%s/%s", currentWorkDir, strZ(repoPath)))); - } - } - - FUNCTION_LOG_RETURN_VOID(); -} - -/**********************************************************************************************************************************/ -void -cfgLoad(unsigned int argListSize, const char *argList[]) -{ - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(UINT, argListSize); - FUNCTION_LOG_PARAM(CHARPY, argList); - FUNCTION_LOG_END(); - - MEM_CONTEXT_TEMP_BEGIN() - { - // Make a copy of the arguments so they can be manipulated - StringList *const argListNew = strLstNew(); - - for (unsigned int argListIdx = 0; argListIdx < argListSize; argListIdx++) - strLstAddZ(argListNew, argList[argListIdx]); - - // Parse config from command line - TRY_BEGIN() - { - cfgParseP(storagePosixNewP(FSLASH_STR), strLstSize(argListNew), strLstPtr(argListNew), .noConfigLoad = true); - } - CATCH(CommandRequiredError) - { - strLstAddZ(argListNew, CFGCMD_BUILD); - cfgParseP(storagePosixNewP(FSLASH_STR), strLstSize(argListNew), strLstPtr(argListNew), .noConfigLoad = true); - } - TRY_END(); - - // Error on noop command. This command is required to hold options that must be declared but are unused by test. - if (cfgCommand() == cfgCmdNoop) - THROW(CommandInvalidError, "invalid command '" CFGCMD_NOOP "'"); - - // Load the log settings - if (!cfgCommandHelp()) - cfgLoadLogSetting(); - - // Neutralize the umask to make the repository file/path modes more consistent - if (cfgOptionValid(cfgOptNeutralUmask) && cfgOptionBool(cfgOptNeutralUmask)) - umask(0000); - - // Set IO buffer size - if (cfgOptionValid(cfgOptBufferSize)) - ioBufferSizeSet(cfgOptionUInt(cfgOptBufferSize)); - - // Update options that have complex rules - cfgLoadUpdateOption(); - - // Begin the command - cmdBegin(); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN_VOID(); -} diff --git a/doc/src/config/load.h b/doc/src/config/load.h deleted file mode 100644 index 082fe1d86..000000000 --- a/doc/src/config/load.h +++ /dev/null @@ -1,13 +0,0 @@ -/*********************************************************************************************************************************** -Documentation Configuration Load -***********************************************************************************************************************************/ -#ifndef DOC_CONFIG_LOAD_H -#define DOC_CONFIG_LOAD_H - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Load the configuration -void cfgLoad(unsigned int argListSize, const char *argList[]); - -#endif diff --git a/doc/src/config/meson.build b/doc/src/config/meson.build deleted file mode 100644 index 29bd0aff0..000000000 --- a/doc/src/config/meson.build +++ /dev/null @@ -1,16 +0,0 @@ -#################################################################################################################################### -# Generate config -#################################################################################################################################### -doc_parse_auto_c_inc = custom_target( - 'doc_parse.auto.c.inc', - output: 'parse.auto.c.inc', - depend_files: [ - '../build/config/config.yaml', - ], - command: [ - build_code, - 'config', - '@CURRENT_SOURCE_DIR@/../..', - '@BUILD_ROOT@/doc', - ], -) diff --git a/doc/src/main.c b/doc/src/main.c deleted file mode 100644 index 123ef9b13..000000000 --- a/doc/src/main.c +++ /dev/null @@ -1,90 +0,0 @@ -/*********************************************************************************************************************************** -Main -***********************************************************************************************************************************/ -#include - -#include - -#include "command/build/build.h" -#include "command/command.h" -#include "command/exit.h" -#include "command/help/help.h" -#include "common/debug.h" -#include "common/log.h" -#include "common/macro.h" -#include "common/memContext.h" -#include "common/stat.h" -#include "config/load.h" -#include "config/parse.h" -#include "storage/posix/storage.h" -#include "version.h" - -/*********************************************************************************************************************************** -Include automatically generated help data -***********************************************************************************************************************************/ -#include "command/help/help.auto.c.inc" - -int -main(int argListSize, const char *argList[]) -{ - // Set stack trace and mem context error cleanup handlers - static const ErrorHandlerFunction errorHandlerList[] = {stackTraceClean, memContextClean}; - errorHandlerSet(errorHandlerList, LENGTH_OF(errorHandlerList)); - - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(INT, argListSize); - FUNCTION_LOG_PARAM(CHARPY, argList); - FUNCTION_LOG_END(); - - // Initialize command with the start time - cmdInit(); - - // Initialize statistics collector - statInit(); - - // Initialize exit handler - exitInit(); - - // Process commands - volatile int result = 0; - volatile bool error = false; - - TRY_BEGIN() - { - // Load the configuration - // ------------------------------------------------------------------------------------------------------------------------- - cfgLoad((unsigned int)argListSize, argList); - - // Handle command - // ------------------------------------------------------------------------------------------------------------------------- - switch (cfgCommandHelp() ? cfgCmdHelp : cfgCommand()) - { - // Build - // ----------------------------------------------------------------------------------------------------------------- - case cfgCmdBuild: - cmdBuild(cfgOptionStr(cfgOptRepoPath), cfgOptionKvNull(cfgOptVar)); - break; - - // Help/version - // ----------------------------------------------------------------------------------------------------------------- - case cfgCmdHelp: - case cfgCmdVersion: - cmdHelp(BUF(helpData, sizeof(helpData))); - break; - - // Error on commands that should have been handled - // ----------------------------------------------------------------------------------------------------------------- - case cfgCmdNoop: - THROW_FMT(AssertError, "'%s' command should have been handled", cfgCommandName()); - break; - } - } - CATCH_FATAL() - { - error = true; - result = exitSafe(result, true, 0); - } - TRY_END(); - - FUNCTION_LOG_RETURN(INT, error ? result : exitSafe(result, false, 0)); -} diff --git a/doc/src/meson.build b/doc/src/meson.build deleted file mode 100644 index 02285f56e..000000000 --- a/doc/src/meson.build +++ /dev/null @@ -1,78 +0,0 @@ -#################################################################################################################################### -# Write configuration -#################################################################################################################################### -configure_file(output: 'build.auto.h', configuration: configuration) - -#################################################################################################################################### -# Build config target -#################################################################################################################################### -# build parse.auto.c.inc -subdir('config') - -#################################################################################################################################### -# Build help target -#################################################################################################################################### -# build help.auto.c.inc -subdir('command/help') - -#################################################################################################################################### -# test target -#################################################################################################################################### -src_doc = [ - '../../src/build/common/json.c', - '../../src/build/common/regExp.c', - '../../src/build/common/render.c', - '../../src/build/common/string.c', - '../../src/build/common/xml.c', - '../../src/build/common/yaml.c', - '../../src/build/config/parse.c', - '../../src/build/help/parse.c', - '../../src/build/help/render.c', - '../../src/command/command.c', - '../../src/command/exit.c', - '../../src/command/help/help.c', - '../../src/command/lock.c', - '../../src/common/compress/common.c', - '../../src/common/compress/bz2/common.c', - '../../src/common/compress/bz2/compress.c', - '../../src/common/compress/bz2/decompress.c', - '../../src/common/ini.c', - '../../src/common/io/fd.c', - '../../src/common/io/fdRead.c', - '../../src/common/io/fdWrite.c', - '../../src/common/fork.c', - '../../src/common/lock.c', - '../../src/common/stat.c', - '../../src/config/config.c', - '../../src/config/parse.c', - 'command/build/build.c', - 'command/build/eval.c', - 'command/build/man.c', - 'command/build/pre.c', - 'command/build/reference.c', - 'command/build/varStore.c', - 'config/load.c', - 'main.c', -] - -executable( - 'doc-pgbackrest', - src_common, - src_doc, - doc_help_auto_c_inc, - doc_parse_auto_c_inc, - include_directories: include_directories('.', '../../src'), - c_args: [ - # Use large buffer sizes to capture large error/log outputs - '-DERROR_MESSAGE_BUFFER_SIZE=262144', - '-DLOG_BUFFER_SIZE=262144', - arg_unity, - ], - dependencies: [ - lib_backtrace, - lib_bz2, - lib_xml, - lib_yaml, - ], - build_by_default: false, -) diff --git a/doc/xml/contributing.xml b/doc/xml/contributing.xml index 21c12d94c..36420164b 100644 --- a/doc/xml/contributing.xml +++ b/doc/xml/contributing.xml @@ -93,8 +93,7 @@ apt-get install rsync git build-essential valgrind libssl-dev zlib1g-dev libxml2-dev libpq-dev pkg-config libxml-checker-perl liblz4-dev liblz4-tool - zstd libzstd-dev bzip2 libbz2-dev libyaml-dev ccache meson python3-yaml - python3-coverage + zstd libzstd-dev bzip2 libbz2-dev ccache meson python3-yaml python3-coverage -y 2>&1 @@ -642,8 +641,8 @@ run 8/1 ------------- L2285 no current backups

To add an option, two files need be to be modified:

- src/build/config/config.yaml - src/build/help/help.xml + build/config.yaml + doc/xml/reference.xml

These files are discussed in the following sections along with how to verify the help command output.

@@ -654,7 +653,7 @@ run 8/1 ------------- L2285 no current backups

There are detailed comment blocks above each section that explain the rules for defining commands and options. Regarding options, there are two types: 1) command line only, and 2) configuration file. With the exception of secrets, all configuration file options can be passed on the command line. To configure an option for the configuration file, the section: key must be present.

-

The option: section is broken into sub-sections by a simple comment divider (e.g. # Repository options) under which the options are organized alphabetically by option name. To better explain this section, two hypothetical examples will be discussed. For more details, see config.yaml.

+

The option: section is broken into sub-sections by a simple comment divider (e.g. # Repository options) under which the options are organized alphabetically by option name. To better explain this section, two hypothetical examples will be discussed. For more details, see config.yaml.

@@ -723,8 +722,8 @@ run 8/1 ------------- L2285 no current backups
-
- help.xml +
+ reference.xml

All options must be documented or the system will error during the build. To add an option, find the command section identified by command id="COMMAND" section where COMMAND is the name of the command (e.g. expire) or, if the option is used by more than one command and the definition for the option is the same for all of the commands, the operation-general title="General Options" section.

diff --git a/src/build/help/help.xml b/doc/xml/reference.xml similarity index 100% rename from src/build/help/help.xml rename to doc/xml/reference.xml diff --git a/doc/xml/user-guide.xml b/doc/xml/user-guide.xml index 32f708df2..7d036711c 100644 --- a/doc/xml/user-guide.xml +++ b/doc/xml/user-guide.xml @@ -322,7 +322,7 @@ RUN rm /etc/apt/apt.conf.d/70debconf && \ apt-get update && \ apt-get install -y --no-install-recommends sudo ssh curl vim gnupg lsb-release iputils-ping ca-certificates \ - tzdata locales libssh2-1-dev unzip git libyaml-dev libxml-checker-perl 2>&1 + tzdata locales libssh2-1-dev unzip git python3-yaml 2>&1 # Install the AWS CLI from the vendor installer since it is no longer packaged for Debian/Ubuntu RUN curl -sL "https://awscli.amazonaws.com/awscli-exe-linux-`uname -m`.zip" -o /tmp/awscliv2.zip && \ @@ -391,7 +391,7 @@ RUN dnf install -y azure-cli # Install dev packages - RUN yum install -y libyaml-devel 2>&1 + RUN dnf install -y python3-pyyaml # Install CA certificate RUN update-ca-trust extract @@ -1085,7 +1085,6 @@ meson setup {[build-meson-path]} {[build-br-path-tmp]} && - ninja -C {[build-meson-path]} src/build-code && mkdir -p {[build-br-path-tmp]}/doc/output/html && mkdir -p {[build-br-path-tmp]}/doc/output/man && touch {[build-br-path-tmp]}/doc/DIST.md && @@ -1095,15 +1094,15 @@ - + - apt-get remove -y libyaml-dev 2>&1 + apt-get remove -y python3-yaml 2>&1 - yum remove -y libyaml-devel 2>&1 + yum remove -y python3-pyyaml 2>&1 diff --git a/meson.build b/meson.build index 449a049ff..cdd3c9994 100644 --- a/meson.build +++ b/meson.build @@ -163,8 +163,8 @@ add_project_arguments(cc.get_supported_arguments('-fmacro-prefix-map=@0@/test/sr #################################################################################################################################### # Detect a distribution tarball, which is marked by the dist file written into the tarball root by meson dist. A distribution -# tarball ships the generated files, so the code generation tooling (build-code, the doc and test tools, libyaml) is not built. A -# normal source tree such as a git checkout has no marker and generates the files. +# tarball ships the generated files, so the code generation tooling (build.py, the doc and test tools) is not used. A normal source +# tree such as a git checkout has no marker and generates the files. #################################################################################################################################### dist = run_command( [ @@ -218,11 +218,6 @@ lib_pq = dependency('libpq') # Find required xml library lib_xml = dependency('libxml-2.0') -# Find yaml library (only used by the code generation tooling, which is not built from a distribution tarball) -if not dist - lib_yaml = dependency('yaml-0.1') -endif - # Find required gz library lib_z = dependency('zlib') @@ -287,10 +282,8 @@ subdir('src') # out all subdirs, which excludes the smoke test from them. subdir('test') -# The doc tool generates code and is not built from a distribution tarball +# A distribution tarball ships the generated files, so it neither generates them nor bakes them in if not dist - subdir('doc/src') - # Bake the generated files and the dist marker into the tarball so it builds without the code generation tooling - meson.add_dist_script('src/build/dist.sh') + meson.add_dist_script('build/dist.sh') endif diff --git a/src/build/common/json.c b/src/build/common/json.c deleted file mode 100644 index eaeed52ff..000000000 --- a/src/build/common/json.c +++ /dev/null @@ -1,21 +0,0 @@ -/*********************************************************************************************************************************** -JSON Extensions -***********************************************************************************************************************************/ -// Include core module -#include "common/type/json.c" - -#include "build/common/json.h" - -/**********************************************************************************************************************************/ -bool -jsonReadUntil(JsonRead *const this, const JsonType type) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(JSON_READ, this); - FUNCTION_TEST_PARAM(ENUM, type); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - - FUNCTION_TEST_RETURN(BOOL, jsonReadTypeNextIgnoreComma(this) != type); -} diff --git a/src/build/common/json.h b/src/build/common/json.h deleted file mode 100644 index 0a02733a8..000000000 --- a/src/build/common/json.h +++ /dev/null @@ -1,15 +0,0 @@ -/*********************************************************************************************************************************** -JSON Extensions -***********************************************************************************************************************************/ -#ifndef BUILD_COMMON_JSON_H -#define BUILD_COMMON_JSON_H - -#include "common/type/json.h" - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Return true unless specified type found -bool jsonReadUntil(JsonRead *this, JsonType type); - -#endif diff --git a/src/build/common/render.c b/src/build/common/render.c deleted file mode 100644 index b452e465a..000000000 --- a/src/build/common/render.c +++ /dev/null @@ -1,52 +0,0 @@ -/*********************************************************************************************************************************** -Build Common -***********************************************************************************************************************************/ -#include - -#include - -#include "build/common/render.h" -#include "common/type/stringId.h" - -/**********************************************************************************************************************************/ -String * -bldStrIdSeq(const char *const buffer, const unsigned int sequence) -{ - StringId strId = strIdSeqFromZ(buffer, sequence); - - if (sequence == STRING_ID_SEQ_NONE) - return strNewFmt("STRID%u(\"%s\", 0x%" PRIx64 ")", (unsigned int)(strId & 1) + 5, buffer, strId); - - return strNewFmt("STRID%uS(\"%s\", %u, 0x%" PRIx64 ")", (unsigned int)(strId & 1) + 5, buffer, strIdSeq(strId), strId); -} - -/**********************************************************************************************************************************/ -String * -bldEnum(const char *const prefix, const String *const value) -{ - String *const result = strNew(); - const char *const valuePtr = strZ(value); - bool upper = false; - - // Add prefix and set first non-prefix letter to upper - if (prefix != NULL) - { - strCatZ(result, prefix); - upper = true; - } - - // Add remaining letters removing dashes and upper-casing the letter after the dash - for (unsigned int valueIdx = 0; valueIdx < strSize(value); valueIdx++) - { - strCatChr(result, upper ? (char)toupper(valuePtr[valueIdx]) : valuePtr[valueIdx]); - upper = false; - - if (valuePtr[valueIdx + 1] == '-') - { - upper = true; - valueIdx++; - } - } - - return result; -} diff --git a/src/build/common/render.h b/src/build/common/render.h deleted file mode 100644 index 513f1c9a5..000000000 --- a/src/build/common/render.h +++ /dev/null @@ -1,95 +0,0 @@ -/*********************************************************************************************************************************** -Build Common -***********************************************************************************************************************************/ -#ifndef BUILD_COMMON_COMMON_H -#define BUILD_COMMON_COMMON_H - -#include - -#include "build/common/string.h" -#include "storage/storage.h" - -/*********************************************************************************************************************************** -Block comments -***********************************************************************************************************************************/ -// {uncrustify_off - comment inside string} -#define COMMENT_BLOCK_BEGIN \ - "/***************************************************************************************************************************" \ - "********" - -#define COMMENT_BLOCK_END \ - "****************************************************************************************************************************" \ - "*******/" - -#define COMMENT_SEPARATOR \ - " // ---------------------------------------------------------------------------------------------------------------------" \ - "--------" -// {uncrustify_on} - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Format a #define with the value aligned at column 69 -FN_INLINE_ALWAYS String * -bldDefineRender(const String *const define, const String *const value) -{ - return strNewFmt("#define %s%*s%s", strZ(define), (int)(60 - strSize(define)), "", strZ(value)); -} - -// Convert identifiers like test-id to testId and add an optional prefix -String *bldEnum(const char *const prefix, const String *const value); - -// Format file header -FN_INLINE_ALWAYS String * -bldHeader(const char *const module, const char *const description) -{ - return strCatFmt( - strNew(), - COMMENT_BLOCK_BEGIN "\n" - "%s\n" - "\n" - "Automatically generated by 'build-code %s' -- do not modify directly.\n" - COMMENT_BLOCK_END "\n", - description, module); -} - -// Put the file if it is different than the existing file -FN_INLINE_ALWAYS void -bldPut(const Storage *const storage, const char *const file, const Buffer *const contentNew) -{ - const Buffer *const contentOld = storageGetP(storageNewReadP(storage, STR(file), .ignoreMissing = true)); - - if (contentOld == NULL || !bufEq(contentOld, contentNew)) - storagePutP(storageNewWriteP(storage, STR(file), .noSyncPath = true), contentNew); -} - -/*********************************************************************************************************************************** -Generate constant StringIds - -To generate a constant StringId call bldStrId(). It will return a String with the generated StringId macro. - -For example: - -bldStrId("test"); - -will return the following: - -STRID5("test", 0xa4cb40) - -which can be used in a function, switch, or #define, e.g.: - -#define TEST_STRID STRID5("test", 0xa4cb40) - -DO NOT MODIFY either parameter in the macro -- ALWAYS use bldStrId() to create a new constant StringId. -***********************************************************************************************************************************/ -// Build StringId define with sequence -String *bldStrIdSeq(const char *buffer, unsigned int sequence); - -// Build StringId define -FN_INLINE_ALWAYS String * -bldStrId(const char *const buffer) -{ - return bldStrIdSeq(buffer, STRING_ID_SEQ_NONE); -} - -#endif diff --git a/src/build/common/xml.c b/src/build/common/xml.c deleted file mode 100644 index a829e803a..000000000 --- a/src/build/common/xml.c +++ /dev/null @@ -1,304 +0,0 @@ -/*********************************************************************************************************************************** -XML Handler Extensions -***********************************************************************************************************************************/ -// Include core module -#include "common/type/xml.c" - -#include "build/common/xml.h" - -/**********************************************************************************************************************************/ -XmlDocument * -xmlDocumentNewParam(const String *const rootNode, const XmlDocumentNewParam param) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(STRING, rootNode); - FUNCTION_TEST_PARAM(STRING, param.dtdName); - FUNCTION_TEST_PARAM(STRING, param.dtdFile); - FUNCTION_TEST_END(); - - ASSERT(rootNode != NULL); - - XmlDocument *const result = xmlDocumentNew(rootNode); - - if (param.dtdName != NULL) - { - ASSERT(param.dtdFile != NULL); - xmlCreateIntSubset( - result->xml, (const uint8_t *)strZ(param.dtdName), NULL, (const uint8_t *)strZ(param.dtdFile)); - } - - FUNCTION_TEST_RETURN(XML_DOCUMENT, result); -} - -/**********************************************************************************************************************************/ -String * -xmlNodeAttribute(const XmlNode *this, const String *name) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(XML_NODE, this); - FUNCTION_TEST_PARAM(STRING, name); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - ASSERT(name != NULL); - - String *result = NULL; - xmlChar *const value = xmlGetProp(this->node, (const uint8_t *)strZ(name)); - - if (value != NULL) - { - result = strNewZ((const char *)value); - xmlFree(value); - } - - FUNCTION_TEST_RETURN(STRING, result); -} - -/**********************************************************************************************************************************/ -void -xmlNodeAttributeRemove(XmlNode *const this, const String *const name) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(XML_NODE, this); - FUNCTION_TEST_PARAM(STRING, name); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - ASSERT(name != NULL); - - xmlUnsetProp(this->node, (const uint8_t *)strZ(name)); - - FUNCTION_TEST_RETURN_VOID(); -} - -/**********************************************************************************************************************************/ -StringList * -xmlNodeAttributeList(const XmlNode *const this) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(XML_NODE, this); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - - StringList *const result = strLstNew(); - - for (xmlAttrPtr currentAttr = this->node->properties; currentAttr != NULL; currentAttr = currentAttr->next) - strLstAddZ(result, (const char *)currentAttr->name); - - FUNCTION_TEST_RETURN(STRING_LIST, result); -} - -/**********************************************************************************************************************************/ -XmlNodeList * -xmlNodeChildListAll(const XmlNode *const this, const bool text) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(XML_NODE, this); - FUNCTION_TEST_PARAM(BOOL, text); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - - XmlNodeList *const list = xmlNodeLstNew(); - - for (xmlNodePtr currentNode = this->node->children; currentNode != NULL; currentNode = currentNode->next) - { - if (text || currentNode->type == XML_ELEMENT_NODE) - xmlNodeLstAdd(list, currentNode); - } - - FUNCTION_TEST_RETURN(XML_NODE_LIST, list); -} - -/**********************************************************************************************************************************/ -void -xmlNodeAttributeSet(XmlNode *const this, const String *const name, const String *const value) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(XML_NODE, this); - FUNCTION_TEST_PARAM(STRING, name); - FUNCTION_TEST_PARAM(STRING, value); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - ASSERT(name != NULL); - ASSERT(value != NULL); - - xmlSetProp(this->node, (const uint8_t *)strZ(name), (const uint8_t *)strZ(value)); - - FUNCTION_TEST_RETURN_VOID(); -} - -/**********************************************************************************************************************************/ -void -xmlNodeChildInsert(XmlNode *const this, const XmlNode *const child) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(XML_NODE, this); - FUNCTION_TEST_PARAM(XML_NODE, child); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - ASSERT(child != NULL); - - MEM_CONTEXT_TEMP_BEGIN() - { - for (xmlNodePtr currentNodeRaw = child->node->children; currentNodeRaw != NULL; currentNodeRaw = currentNodeRaw->next) - { - // Skip comments - if (currentNodeRaw->type == XML_COMMENT_NODE) - continue; - - xmlNodePtr node = xmlDocCopyNode(currentNodeRaw, this->node->doc, 1); - xmlAddPrevSibling(this->node, node); - } - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_TEST_RETURN_VOID(); -} - -/**********************************************************************************************************************************/ -void -xmlNodeChildReplace(XmlNode *const this, const XmlNode *const child) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(XML_NODE, this); - FUNCTION_TEST_PARAM(XML_NODE, child); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - ASSERT(child != NULL); - - xmlNodeChildInsert(this, child); - xmlUnlinkNode(this->node); - xmlFreeNode(this->node); - - FUNCTION_TEST_RETURN_VOID(); -} - -/**********************************************************************************************************************************/ -void -xmlNodeChildAdd(XmlNode *const this, const XmlNode *const child) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(XML_NODE, this); - FUNCTION_TEST_PARAM(XML_NODE, child); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - ASSERT(child != NULL); - - MEM_CONTEXT_TEMP_BEGIN() - { - for (xmlNodePtr currentNodeRaw = child->node->children; currentNodeRaw != NULL; currentNodeRaw = currentNodeRaw->next) - { - // Skip comments - if (currentNodeRaw->type == XML_COMMENT_NODE) - continue; - - // Copy all child nodes (only node and text types are copied) - XmlNode *const currentNode = xmlNodeNew(currentNodeRaw); - - if (currentNode->node->type == XML_ELEMENT_NODE) - { - XmlNode *const node = xmlNodeAdd(this, STR((const char *)currentNode->node->name)); - - // Copy node attributes - for (xmlAttrPtr currentAttr = currentNode->node->properties; currentAttr != NULL; currentAttr = currentAttr->next) - { - xmlNodeAttributeSet( - node, STR((const char *)currentAttr->name), - xmlNodeAttribute(currentNode, STR((const char *)currentAttr->name))); - } - - // Recurse to copy child nodes - xmlNodeChildAdd(node, currentNode); - } - else - { - CHECK_FMT( - AssertError, currentNode->node->type == XML_TEXT_NODE, "unknown type %u in node '%s'", currentNode->node->type, - currentNode->node->name); - - xmlNodeContentSet(this, xmlNodeContent(currentNode)); - } - } - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_TEST_RETURN_VOID(); -} - -/**********************************************************************************************************************************/ -void -xmlNodeTextSet(XmlNode *const this, const String *const content) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(XML_NODE, this); - FUNCTION_TEST_PARAM(STRING, content); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - ASSERT(content != NULL); - - xmlNodeSetContent(this->node, (const xmlChar *)strZ(content)); - - FUNCTION_TEST_RETURN_VOID(); -} - -/*********************************************************************************************************************************** -Free node -***********************************************************************************************************************************/ -static void -xmlNodeFreeResource(THIS_VOID) -{ - THIS(XmlNode); - - FUNCTION_LOG_BEGIN(logLevelTrace); - FUNCTION_LOG_PARAM(XML_NODE, this); - FUNCTION_LOG_END(); - - ASSERT(this != NULL); - - xmlFreeNode(this->node); - - FUNCTION_LOG_RETURN_VOID(); -} - -/**********************************************************************************************************************************/ -XmlNode * -xmlNodeDup(const XmlNode *const node) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(XML_NODE, node); - FUNCTION_TEST_END(); - - OBJ_NEW_BEGIN(XmlNode, .callbackQty = 1) - { - *this = (XmlNode) - { - .node = xmlDocCopyNode(node->node, NULL, 1), - }; - - memContextCallbackSet(objMemContext(this), xmlNodeFreeResource, this); - } - OBJ_NEW_END(); - - FUNCTION_TEST_RETURN(XML_NODE, this); -} - -/**********************************************************************************************************************************/ -void -xmlNodeRemove(XmlNode *const this) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(XML_NODE, this); - FUNCTION_TEST_END(); - - xmlUnlinkNode(this->node); - xmlFreeNode(this->node); - - FUNCTION_TEST_RETURN_VOID(); -} diff --git a/src/build/common/xml.h b/src/build/common/xml.h deleted file mode 100644 index cd8c6ea24..000000000 --- a/src/build/common/xml.h +++ /dev/null @@ -1,63 +0,0 @@ -/*********************************************************************************************************************************** -XML Handler Extensions -***********************************************************************************************************************************/ -#ifndef BUILD_COMMON_XML_H -#define BUILD_COMMON_XML_H - -#include "common/type/xml.h" - -/*********************************************************************************************************************************** -Constructors -***********************************************************************************************************************************/ -typedef struct XmlDocumentNewParam -{ - VAR_PARAM_HEADER; - const String *dtdName; // DTD name, if any - const String *dtdFile; // DTD file (must be set if dtdName is set) -} XmlDocumentNewParam; - -#define xmlDocumentNewP(rootNode, ...) \ - xmlDocumentNewParam(rootNode, (XmlDocumentNewParam){VAR_PARAM_INIT, __VA_ARGS__}) - -XmlDocument *xmlDocumentNewParam(const String *rootNode, XmlDocumentNewParam param); - -/*********************************************************************************************************************************** -Node Getters/Setters -***********************************************************************************************************************************/ -// Get node attribute -String *xmlNodeAttribute(const XmlNode *this, const String *name); - -// Remove node attribute -void xmlNodeAttributeRemove(XmlNode *this, const String *name); - -// Set node attribute -void xmlNodeAttributeSet(XmlNode *this, const String *name, const String *value); - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Add all child nodes to another node -void xmlNodeChildAdd(XmlNode *this, const XmlNode *child); - -// Insert all child nodes before another node -void xmlNodeChildInsert(XmlNode *this, const XmlNode *child); - -// Replace a node with the child nodes -void xmlNodeChildReplace(XmlNode *this, const XmlNode *child); - -// Get list of attribute names -StringList *xmlNodeAttributeList(const XmlNode *this); - -// Get list of all child nodes -XmlNodeList *xmlNodeChildListAll(const XmlNode *this, bool text); - -// Set text node content (replaces existing content, unlike xmlNodeContentSet which appends) -void xmlNodeTextSet(XmlNode *this, const String *content); - -// Duplicate node -XmlNode *xmlNodeDup(const XmlNode *this); - -// Remove node -void xmlNodeRemove(XmlNode *this); - -#endif diff --git a/src/build/common/yaml.c b/src/build/common/yaml.c deleted file mode 100644 index 644b2908e..000000000 --- a/src/build/common/yaml.c +++ /dev/null @@ -1,277 +0,0 @@ -/*********************************************************************************************************************************** -Yaml Handler -***********************************************************************************************************************************/ -#include - -#include - -#include "common/debug.h" -#include "common/log.h" - -#include "build/common/yaml.h" - -/*********************************************************************************************************************************** -Object type -***********************************************************************************************************************************/ -struct Yaml -{ - yaml_parser_t parser; // Parse context - bool eventNextSet; // Is the next event set? - YamlEvent eventNext; // Next event, stored after a peek -}; - -/*********************************************************************************************************************************** -Free parser context -***********************************************************************************************************************************/ -static void -yamlFreeResource(THIS_VOID) -{ - THIS(Yaml); - - FUNCTION_LOG_BEGIN(logLevelTrace); - FUNCTION_LOG_PARAM(YAML, this); - FUNCTION_LOG_END(); - - ASSERT(this != NULL); - - yaml_parser_delete(&this->parser); - - FUNCTION_LOG_RETURN_VOID(); -} - -/**********************************************************************************************************************************/ -Yaml * -yamlNew(const Buffer *const buffer) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(BUFFER, buffer); - FUNCTION_TEST_END(); - - OBJ_NEW_BEGIN(Yaml, .childQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1) - { - *this = (Yaml){{0}}; // Extra braces are required for older gcc versions - - // Initialize parser context - CHECK(ServiceError, yaml_parser_initialize(&this->parser), "unable to initialize yaml parser"); - memContextCallbackSet(objMemContext(this), yamlFreeResource, this); - - // Set yaml string - yaml_parser_set_input_string(&this->parser, bufPtrConst(buffer), bufUsed(buffer)); - - // Start document - CHECK(FormatError, yamlEventNext(this).type == yamlEventTypeStreamBegin, "expected yaml stream begin"); - CHECK(FormatError, yamlEventNext(this).type == yamlEventTypeDocBegin, "expected yaml document begin"); - } - OBJ_NEW_END(); - - FUNCTION_TEST_RETURN_TYPE_P(Yaml, this); -} - -/**********************************************************************************************************************************/ -// Helper to map event type -static YamlEventType -yamlEventType(yaml_event_type_t type) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(ENUM, type); - FUNCTION_TEST_END(); - - switch (type) - { - case YAML_STREAM_START_EVENT: - FUNCTION_TEST_RETURN_TYPE(YamlEventType, yamlEventTypeStreamBegin); - - case YAML_STREAM_END_EVENT: - FUNCTION_TEST_RETURN_TYPE(YamlEventType, yamlEventTypeStreamEnd); - - case YAML_DOCUMENT_START_EVENT: - FUNCTION_TEST_RETURN_TYPE(YamlEventType, yamlEventTypeDocBegin); - - case YAML_DOCUMENT_END_EVENT: - FUNCTION_TEST_RETURN_TYPE(YamlEventType, yamlEventTypeDocEnd); - - case YAML_ALIAS_EVENT: - FUNCTION_TEST_RETURN_TYPE(YamlEventType, yamlEventTypeAlias); - - case YAML_SCALAR_EVENT: - FUNCTION_TEST_RETURN_TYPE(YamlEventType, yamlEventTypeScalar); - - case YAML_SEQUENCE_START_EVENT: - FUNCTION_TEST_RETURN_TYPE(YamlEventType, yamlEventTypeSeqBegin); - - case YAML_SEQUENCE_END_EVENT: - FUNCTION_TEST_RETURN_TYPE(YamlEventType, yamlEventTypeSeqEnd); - - case YAML_MAPPING_START_EVENT: - FUNCTION_TEST_RETURN_TYPE(YamlEventType, yamlEventTypeMapBegin); - - case YAML_MAPPING_END_EVENT: - FUNCTION_TEST_RETURN_TYPE(YamlEventType, yamlEventTypeMapEnd); - - default: - CHECK(FormatError, type == YAML_NO_EVENT, "expected yaml no event"); - FUNCTION_TEST_RETURN_TYPE(YamlEventType, yamlEventTypeNone); - } -} - -YamlEvent -yamlEventPeek(Yaml *const this) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(YAML, this); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - - if (!this->eventNextSet) - { - yaml_event_t event; - - if (!yaml_parser_parse(&this->parser, &event)) - { - // These should always be set - CHECK(ServiceError, this->parser.problem_mark.line && this->parser.problem_mark.column, "invalid yaml error info"); - - THROW_FMT( - FormatError, "yaml parse error: %s at line: %lu column: %lu", this->parser.problem, - (unsigned long)this->parser.problem_mark.line + 1, (unsigned long)this->parser.problem_mark.column + 1); - } - - this->eventNext = (YamlEvent) - { - .type = yamlEventType(event.type), - .line = event.start_mark.line + 1, - .column = event.start_mark.column + 1, - }; - - if (this->eventNext.type == yamlEventTypeScalar) - { - MEM_CONTEXT_OBJ_BEGIN(this) - { - this->eventNext.value = strNewZ((const char *)event.data.scalar.value); - } - MEM_CONTEXT_OBJ_END(); - } - - yaml_event_delete(&event); - - this->eventNextSet = true; - } - - FUNCTION_TEST_RETURN_TYPE(YamlEvent, this->eventNext); -} - -YamlEvent -yamlEventNext(Yaml *this) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(YAML, this); - FUNCTION_TEST_END(); - - ASSERT(this != NULL); - - if (!this->eventNextSet) - yamlEventPeek(this); - - this->eventNextSet = false; - - FUNCTION_TEST_RETURN_TYPE(YamlEvent, this->eventNext); -} - -/**********************************************************************************************************************************/ -YamlEvent -yamlEventNextCheck(Yaml *this, YamlEventType type) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(YAML, this); - FUNCTION_TEST_PARAM(STRING_ID, type); - FUNCTION_TEST_END(); - - YamlEvent result = yamlEventNext(this); - yamlEventCheck(result, type); - - FUNCTION_TEST_RETURN_TYPE(YamlEvent, result); -} - -/**********************************************************************************************************************************/ -YamlEvent -yamlEventCheck(YamlEvent event, YamlEventType type) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(YAML_EVENT, event); - FUNCTION_TEST_PARAM(STRING_ID, type); - FUNCTION_TEST_END(); - - if (event.type != type) - { - if (event.type == yamlEventTypeScalar) - { - THROW_FMT( - FormatError, "expected event type '%s' but got scalar '%s' at line %zu, column %zu", zNewStrId(type), - strZ(event.value), event.line, event.column); - } - else - { - THROW_FMT( - FormatError, "expected event type '%s' but got '%s' at line %zu, column %zu", zNewStrId(type), - zNewStrId(event.type), event.line, event.column); - } - } - - FUNCTION_TEST_RETURN_TYPE(YamlEvent, event); -} - -/**********************************************************************************************************************************/ -void -yamlScalarCheck(const YamlEvent event, const String *const value) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(YAML_EVENT, event); - FUNCTION_TEST_PARAM(STRING, value); - FUNCTION_TEST_END(); - - if (event.type != yamlEventTypeScalar) - { - THROW_FMT( - FormatError, "expected scalar '%s' but got event type '%s' at line %zu, column %zu", strZ(value), - zNewStrId(event.type), event.line, event.column); - } - - if (!strEq(event.value, value)) - { - THROW_FMT( - FormatError, "expected scalar '%s' but got '%s' at line %zu, column %zu", strZ(value), strZ(event.value), event.line, - event.column); - } - - FUNCTION_TEST_RETURN_VOID(); -} - -void -yamlScalarNextCheck(Yaml *const this, const String *const value) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(YAML, this); - FUNCTION_TEST_PARAM(STRING, value); - FUNCTION_TEST_END(); - - yamlScalarCheck(yamlEventNext(this), value); - - FUNCTION_TEST_RETURN_VOID(); -} - -/**********************************************************************************************************************************/ -bool -yamlBoolParse(YamlEvent event) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(YAML_EVENT, event); - FUNCTION_TEST_END(); - - if (strEq(event.value, FALSE_STR)) - FUNCTION_TEST_RETURN(BOOL, false); - else if (strEq(event.value, TRUE_STR)) - FUNCTION_TEST_RETURN(BOOL, true); - - THROW_FMT(FormatError, "invalid boolean '%s'", strZ(event.value)); -} diff --git a/src/build/common/yaml.h b/src/build/common/yaml.h deleted file mode 100644 index ad3b4e432..000000000 --- a/src/build/common/yaml.h +++ /dev/null @@ -1,163 +0,0 @@ -/*********************************************************************************************************************************** -Yaml Handler -***********************************************************************************************************************************/ -#ifndef COMMON_TYPE_YAML_H -#define COMMON_TYPE_YAML_H - -#include - -/*********************************************************************************************************************************** -Yaml object -***********************************************************************************************************************************/ -typedef struct Yaml Yaml; - -#include "common/memContext.h" -#include "common/type/buffer.h" -#include "common/type/object.h" -#include "common/type/stringId.h" - -/*********************************************************************************************************************************** -Yaml event type -***********************************************************************************************************************************/ -typedef enum -{ - yamlEventTypeNone = STRID5("none", 0x2b9ee0), - yamlEventTypeStreamBegin = STRID5("stream-begin", 0x724e516da12ca930), - yamlEventTypeStreamEnd = STRID5("stream-end", 0x8e2eda12ca930), - yamlEventTypeDocBegin = STRID5("doc-begin", 0xe49ca2d8de40), - yamlEventTypeDocEnd = STRID5("doc-end", 0x11c5d8de40), - yamlEventTypeAlias = STRID5("alias", 0x130a5810), - yamlEventTypeScalar = STRID5("scalar", 0x241604730), - yamlEventTypeSeqBegin = STRID5("seq-begin", 0xe49ca2dc4b30), - yamlEventTypeSeqEnd = STRID5("seq-end", 0x11c5dc4b30), - yamlEventTypeMapBegin = STRID5("map-begin", 0xe49ca2dc02d0), - yamlEventTypeMapEnd = STRID5("map-end", 0x11c5dc02d0), -} YamlEventType; - -/*********************************************************************************************************************************** -Yaml event -***********************************************************************************************************************************/ -typedef struct YamlEvent -{ - YamlEventType type; // Type (e.g. scalar) - const String *value; // Value, when type is scalar - size_t line; // Parse line - size_t column; // Parse column -} YamlEvent; - -/*********************************************************************************************************************************** -Constructors -***********************************************************************************************************************************/ -Yaml *yamlNew(const Buffer *const buffer); - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Get next event from doc -YamlEvent yamlEventNext(Yaml *this); - -// Get next event from doc and check the type -YamlEvent yamlEventNextCheck(Yaml *this, YamlEventType type); - -// Check the event type -YamlEvent yamlEventCheck(YamlEvent event, YamlEventType type); - -// Peek at the next event -YamlEvent yamlEventPeek(Yaml *this); - -// Is the next event of the given type? -FN_INLINE_ALWAYS bool -yamlEventPeekIs(Yaml *const this, const YamlEventType type) -{ - return yamlEventPeek(this).type == type; -} - -// Get next scalar -FN_INLINE_ALWAYS YamlEvent -yamlScalarNext(Yaml *const this) -{ - return yamlEventNextCheck(this, yamlEventTypeScalar); -} - -// Check scalar -void yamlScalarCheck(YamlEvent event, const String *value); - -FN_INLINE_ALWAYS void -yamlScalarCheckZ(const YamlEvent event, const char *const value) -{ - yamlScalarCheck(event, STR(value)); -} - -void yamlScalarNextCheck(Yaml *this, const String *value); - -FN_INLINE_ALWAYS void -yamlScalarNextCheckZ(Yaml *const this, const char *const value) -{ - yamlScalarNextCheck(this, STR(value)); -} - -// Consume an empty map (map begin immediately followed by map end) -FN_INLINE_ALWAYS void -yamlMapEmpty(Yaml *const this) -{ - yamlEventNextCheck(this, yamlEventTypeMapBegin); - yamlEventNextCheck(this, yamlEventTypeMapEnd); -} - -// Convert an event to a boolean (or error) -bool yamlBoolParse(YamlEvent event); - -/*********************************************************************************************************************************** -Destructor -***********************************************************************************************************************************/ -FN_INLINE_ALWAYS void -yamlFree(Yaml *const this) -{ - objFree(this); -} - -/*********************************************************************************************************************************** -Helper macros for iterating sequences and maps -***********************************************************************************************************************************/ -#define YAML_ITER_BEGIN(yamlParam, eventBegin, eventEnd) \ - do \ - { \ - Yaml *const YAML_ITER_yaml = yamlParam; \ - yamlEventNextCheck(YAML_ITER_yaml, eventBegin); \ - \ - while (yamlEventPeek(YAML_ITER_yaml).type != eventEnd) \ - { - -#define YAML_ITER_END(eventEnd) \ - } \ - \ - yamlEventNextCheck(YAML_ITER_yaml, eventEnd); \ - } \ - while (0) - -#define YAML_SEQ_BEGIN(yaml) \ - YAML_ITER_BEGIN(yaml, yamlEventTypeSeqBegin, yamlEventTypeSeqEnd) - -#define YAML_SEQ_END() \ - YAML_ITER_END(yamlEventTypeSeqEnd); - -#define YAML_MAP_BEGIN(yaml) \ - YAML_ITER_BEGIN(yaml, yamlEventTypeMapBegin, yamlEventTypeMapEnd) - -#define YAML_MAP_END() \ - YAML_ITER_END(yamlEventTypeMapEnd); - -/*********************************************************************************************************************************** -Macros for function logging -***********************************************************************************************************************************/ -#define FUNCTION_LOG_YAML_TYPE \ - Yaml * -#define FUNCTION_LOG_YAML_FORMAT(value, buffer, bufferSize) \ - objNameToLog(value, "Yaml", buffer, bufferSize) - -#define FUNCTION_LOG_YAML_EVENT_TYPE \ - YamlEvent -#define FUNCTION_LOG_YAML_EVENT_FORMAT(value, buffer, bufferSize) \ - objNameToLog(&value, "YamlEvent", buffer, bufferSize) - -#endif diff --git a/src/build/config/parse.c b/src/build/config/parse.c deleted file mode 100644 index e9661b4af..000000000 --- a/src/build/config/parse.c +++ /dev/null @@ -1,1321 +0,0 @@ -/*********************************************************************************************************************************** -Parse Configuration Yaml -***********************************************************************************************************************************/ -#include - -#include - -#include "common/log.h" -#include "common/type/convert.h" -#include "storage/posix/storage.h" - -#include "build/common/yaml.h" -#include "build/config/parse.h" - -/*********************************************************************************************************************************** -Command role constants -***********************************************************************************************************************************/ -STRING_EXTERN(CMD_ROLE_ASYNC_STR, CMD_ROLE_ASYNC); -STRING_EXTERN(CMD_ROLE_LOCAL_STR, CMD_ROLE_LOCAL); -STRING_EXTERN(CMD_ROLE_MAIN_STR, CMD_ROLE_MAIN); -STRING_EXTERN(CMD_ROLE_REMOTE_STR, CMD_ROLE_REMOTE); - -/*********************************************************************************************************************************** -Command constants -***********************************************************************************************************************************/ -STRING_EXTERN(CMD_HELP_STR, CMD_HELP); -STRING_EXTERN(CMD_VERSION_STR, CMD_VERSION); - -/*********************************************************************************************************************************** -Option type constants -***********************************************************************************************************************************/ -STRING_EXTERN(OPT_TYPE_BOOLEAN_STR, OPT_TYPE_BOOLEAN); -STRING_EXTERN(OPT_TYPE_HASH_STR, OPT_TYPE_HASH); -STRING_EXTERN(OPT_TYPE_INTEGER_STR, OPT_TYPE_INTEGER); -STRING_EXTERN(OPT_TYPE_LIST_STR, OPT_TYPE_LIST); -STRING_EXTERN(OPT_TYPE_PATH_STR, OPT_TYPE_PATH); -STRING_EXTERN(OPT_TYPE_SIZE_STR, OPT_TYPE_SIZE); -STRING_EXTERN(OPT_TYPE_STRING_STR, OPT_TYPE_STRING); -STRING_EXTERN(OPT_TYPE_STRING_ID_STR, OPT_TYPE_STRING_ID); -STRING_EXTERN(OPT_TYPE_TIME_STR, OPT_TYPE_TIME); - -/*********************************************************************************************************************************** -Option constants -***********************************************************************************************************************************/ -STRING_EXTERN(OPT_BETA_STR, OPT_BETA); -STRING_EXTERN(OPT_STANZA_STR, OPT_STANZA); - -/*********************************************************************************************************************************** -Section constants -***********************************************************************************************************************************/ -STRING_EXTERN(SECTION_COMMAND_LINE_STR, SECTION_COMMAND_LINE); -STRING_EXTERN(SECTION_GLOBAL_STR, SECTION_GLOBAL); -STRING_EXTERN(SECTION_STANZA_STR, SECTION_STANZA); - -/*********************************************************************************************************************************** -Parse command list -***********************************************************************************************************************************/ -typedef struct BldCfgCommandRaw -{ - const String *const name; // See BldCfgCommand for comments - bool internal; - bool logFile; - const String *logLevelDefault; - bool lockRequired; - bool lockRemoteRequired; - const String *lockType; - bool parameterAllowed; - StringList *roleList; -} BldCfgCommandRaw; - -// Helper to parse command roles -static StringList * -bldCfgParseCommandRole(Yaml *const yaml) -{ - StringList *const result = strLstNew(); - - MEM_CONTEXT_TEMP_BEGIN() - { - YAML_MAP_BEGIN(yaml) - { - strLstAdd(result, yamlScalarNext(yaml).value); - - yamlMapEmpty(yaml); - } - YAML_MAP_END(); - } - MEM_CONTEXT_TEMP_END(); - - return result; -} - -static List * -bldCfgParseCommandList(Yaml *const yaml) -{ - List *const result = lstNewP(sizeof(BldCfgCommand), .comparator = lstComparatorStr); - - MEM_CONTEXT_TEMP_BEGIN() - { - yamlEventNextCheck(yaml, yamlEventTypeScalar); - - YAML_MAP_BEGIN(yaml) - { - BldCfgCommandRaw cmdRaw = - { - .name = yamlScalarNext(yaml).value, - .logFile = true, - .logLevelDefault = strNewZ("info"), - .lockType = strNewZ("none"), - }; - - YAML_MAP_BEGIN(yaml) - { - const String *const cmdDef = yamlScalarNext(yaml).value; - - if (strEqZ(cmdDef, "command-role")) - cmdRaw.roleList = bldCfgParseCommandRole(yaml); - else - { - const YamlEvent cmdDefVal = yamlScalarNext(yaml); - - if (strEqZ(cmdDef, "internal")) - cmdRaw.internal = yamlBoolParse(cmdDefVal); - else if (strEqZ(cmdDef, "lock-type")) - cmdRaw.lockType = cmdDefVal.value; - else if (strEqZ(cmdDef, "lock-remote-required")) - cmdRaw.lockRemoteRequired = yamlBoolParse(cmdDefVal); - else if (strEqZ(cmdDef, "lock-required")) - cmdRaw.lockRequired = yamlBoolParse(cmdDefVal); - else if (strEqZ(cmdDef, "log-file")) - cmdRaw.logFile = yamlBoolParse(cmdDefVal); - else if (strEqZ(cmdDef, "log-level-default")) - cmdRaw.logLevelDefault = strLower(strDup(cmdDefVal.value)); - else if (strEqZ(cmdDef, "parameter-allowed")) - cmdRaw.parameterAllowed = yamlBoolParse(cmdDefVal); - else - THROW_FMT(FormatError, "unknown command definition '%s'", strZ(cmdDef)); - } - } - YAML_MAP_END(); - - // Create role list if not defined - if (cmdRaw.roleList == NULL) - cmdRaw.roleList = strLstNew(); - - // Add main to the role list and resort - strLstAddIfMissing(cmdRaw.roleList, CMD_ROLE_MAIN_STR); - strLstSort(cmdRaw.roleList, sortOrderAsc); - - MEM_CONTEXT_BEGIN(lstMemContext(result)) - { - const BldCfgCommand bldCfgCommand = - { - .name = strDup(cmdRaw.name), - .internal = cmdRaw.internal, - .logFile = cmdRaw.logFile, - .logLevelDefault = strDup(cmdRaw.logLevelDefault), - .lockRequired = cmdRaw.lockRequired, - .lockRemoteRequired = cmdRaw.lockRemoteRequired, - .lockType = strDup(cmdRaw.lockType), - .parameterAllowed = cmdRaw.parameterAllowed, - .roleList = strLstDup(cmdRaw.roleList) - }; - - lstAdd(result, &bldCfgCommand); - } - MEM_CONTEXT_END(); - } - YAML_MAP_END(); - - lstSort(result, sortOrderAsc); - } - MEM_CONTEXT_TEMP_END(); - - return result; -} - -/*********************************************************************************************************************************** -Parse option group list -***********************************************************************************************************************************/ -typedef struct BldCfgOptionGroupRaw -{ - const String *const name; // See BldCfgOptionGroup for comments -} BldCfgOptionGroupRaw; - -static List * -bldCfgParseOptionGroupList(Yaml *const yaml) -{ - List *const result = lstNewP(sizeof(BldCfgOptionGroup), .comparator = lstComparatorStr); - - MEM_CONTEXT_TEMP_BEGIN() - { - yamlEventNextCheck(yaml, yamlEventTypeScalar); - - YAML_MAP_BEGIN(yaml) - { - BldCfgOptionGroupRaw optGrpRaw = {.name = yamlScalarNext(yaml).value}; - - yamlMapEmpty(yaml); - - MEM_CONTEXT_BEGIN(lstMemContext(result)) - { - lstAdd(result, &(BldCfgOptionGroup){.name = strDup(optGrpRaw.name)}); - } - MEM_CONTEXT_END(); - } - YAML_MAP_END(); - - lstSort(result, sortOrderAsc); - } - MEM_CONTEXT_TEMP_END(); - - return result; -} - -/*********************************************************************************************************************************** -Parse option list -***********************************************************************************************************************************/ -typedef struct BldCfgOptionAllowRangeMapRaw // See BldCfgOptionAllowRangeMap for comments -{ - const String *map; - const String *min; - const String *max; -} BldCfgOptionAllowRangeMapRaw; - -typedef struct BldCfgOptionAllowRangeRaw // See BldCfgOptionAllowRange for comments -{ - const String *min; - const String *max; - List *mapList; -} BldCfgOptionAllowRangeRaw; - -typedef struct BldCfgOptionDefaultMapRaw -{ - const String *map; // See BldCfgOptionDefaultMap for comments - const String *value; -} BldCfgOptionDefaultMapRaw; - -typedef struct BldCfgOptionDefaultRaw -{ - const String *value; // See BldCfgOptionDefault for comments - List *mapList; -} BldCfgOptionDefaultRaw; - -typedef struct BldCfgOptionDependRaw -{ - const String *option; // See BldCfgOptionDepend for comments - const String *defaultValue; - const StringList *valueList; -} BldCfgOptionDependRaw; - -typedef struct BldCfgOptionDeprecateRaw -{ - const String *name; // See BldCfgOptionDeprecate for comments - bool indexed; - bool unindexed; -} BldCfgOptionDeprecateRaw; - -typedef struct BldCfgOptionCommandRaw -{ - const String *name; // See BldCfgOptionCommand for comments - const Variant *internal; - const Variant *required; - const Variant *sequence; - const BldCfgOptionDefaultRaw *defaultValue; - const BldCfgOptionDependRaw *depend; - const List *allowList; - const StringList *roleList; -} BldCfgOptionCommandRaw; - -typedef struct BldCfgOptionRaw -{ - const String *name; // See BldCfgOption for comments - const String *type; - const String *section; - bool boolLike; - bool internal; - bool beta; - const Variant *required; - const Variant *negate; - bool reset; - bool sequence; - DefaultType defaultType; - const BldCfgOptionDefaultRaw *defaultValue; - const String *group; - bool secure; - const BldCfgOptionDependRaw *depend; - const List *allowList; - const BldCfgOptionAllowRangeRaw *allowRange; - const List *cmdList; - const StringList *cmdRoleList; - const List *deprecateList; -} BldCfgOptionRaw; - -// Helper to parse allow list -static List * -bldCfgParseAllowListDup(const List *const allowList) -{ - List *result = NULL; - - if (allowList != NULL) - { - result = lstNewP(sizeof(BldCfgOptionValue), .comparator = lstComparatorStr); - - for (unsigned int valueIdx = 0; valueIdx < lstSize(allowList); valueIdx++) - { - const BldCfgOptionValue *const bldCfgOptionValue = lstGet(allowList, valueIdx); - const BldCfgOptionValue bldCfgOptionValueCopy = - { - .value = strDup(bldCfgOptionValue->value), - .condition = strDup(bldCfgOptionValue->condition), - }; - - lstAdd(result, &bldCfgOptionValueCopy); - } - } - - return result; -} - -static const List * -bldCfgParseAllowList(Yaml *const yaml, const List *const optList) -{ - List *result = NULL; - - MEM_CONTEXT_TEMP_BEGIN() - { - // If allow list is defined - if (yamlEventPeekIs(yaml, yamlEventTypeSeqBegin)) - { - MEM_CONTEXT_PRIOR_BEGIN() - { - result = lstNewP(sizeof(BldCfgOptionValue), .comparator = lstComparatorStr); - } - MEM_CONTEXT_PRIOR_END(); - - YAML_SEQ_BEGIN(yaml) - { - BldCfgOptionValue bldCfgOptionValue; - - // A scalar is a bare value; a map is a value with a condition - if (yamlEventPeekIs(yaml, yamlEventTypeScalar)) - { - bldCfgOptionValue.value = yamlScalarNext(yaml).value; - bldCfgOptionValue.condition = NULL; - } - else - { - yamlEventNextCheck(yaml, yamlEventTypeMapBegin); - - bldCfgOptionValue.value = yamlScalarNext(yaml).value; - bldCfgOptionValue.condition = yamlScalarNext(yaml).value; - - yamlEventNextCheck(yaml, yamlEventTypeMapEnd); - } - - lstAdd(result, &bldCfgOptionValue); - } - YAML_SEQ_END(); - } - // Else allow list is inherited - else - { - CHECK(AssertError, optList != NULL, "option list is NULL"); - const YamlEvent allowListVal = yamlEventNextCheck(yaml, yamlEventTypeScalar); - - const BldCfgOptionRaw *const optInherit = lstFind(optList, &allowListVal.value); - CHECK(AssertError, optInherit != NULL, "inherited option is NULL"); - - MEM_CONTEXT_PRIOR_BEGIN() - { - result = bldCfgParseAllowListDup(optInherit->allowList); - } - MEM_CONTEXT_PRIOR_END(); - } - - CHECK(AssertError, result != NULL, "result is NULL"); - } - MEM_CONTEXT_TEMP_END(); - - return result; -} - -// Helper to parse allow range -static const BldCfgOptionAllowRange * -bldCfgParseAllowRangeDup(const BldCfgOptionAllowRangeRaw *const allowRangeRaw) -{ - BldCfgOptionAllowRange *result = NULL; - - if (allowRangeRaw != NULL) - { - result = memNew(sizeof(BldCfgOptionAllowRange)); - *result = (BldCfgOptionAllowRange){0}; - - result->min = strDup(allowRangeRaw->min); - result->max = strDup(allowRangeRaw->max); - - if (allowRangeRaw->mapList != NULL) - { - List *const mapList = lstNewP(sizeof(BldCfgOptionAllowRangeMap), .comparator = lstComparatorStr); - - for (unsigned int mapIdx = 0; mapIdx < lstSize(allowRangeRaw->mapList); mapIdx++) - { - const BldCfgOptionAllowRangeMapRaw *const allowRangeMapRaw = lstGet(allowRangeRaw->mapList, mapIdx); - const BldCfgOptionAllowRangeMap allowRangeMap = - { - .map = strDup(allowRangeMapRaw->map), - .min = strDup(allowRangeMapRaw->min), - .max = strDup(allowRangeMapRaw->max), - }; - - lstAdd(mapList, &allowRangeMap); - } - - result->mapList = mapList; - } - } - - return result; -} - -static const BldCfgOptionAllowRangeRaw * -bldCfgParseAllowRange(Yaml *const yaml) -{ - BldCfgOptionAllowRangeRaw *result = memNew(sizeof(BldCfgOptionAllowRangeRaw)); - *result = (BldCfgOptionAllowRangeRaw){0}; - - MEM_CONTEXT_TEMP_BEGIN() - { - yamlEventNextCheck(yaml, yamlEventTypeSeqBegin); - - // If the range is a simple [min, max] pair - if (yamlEventPeekIs(yaml, yamlEventTypeScalar)) - { - const YamlEvent allowRangeMinVal = yamlScalarNext(yaml); - const YamlEvent allowRangeMaxVal = yamlScalarNext(yaml); - - MEM_CONTEXT_PRIOR_BEGIN() - { - result->min = strDup(allowRangeMinVal.value); - result->max = strDup(allowRangeMaxVal.value); - } - MEM_CONTEXT_PRIOR_END(); - - yamlEventNextCheck(yaml, yamlEventTypeSeqEnd); - } - // Else the range is a list of per-map [min, max] pairs - else - { - MEM_CONTEXT_PRIOR_BEGIN() - { - result->mapList = lstNewP(sizeof(BldCfgOptionAllowRangeMapRaw)); - } - MEM_CONTEXT_PRIOR_END(); - - while (yamlEventPeek(yaml).type != yamlEventTypeSeqEnd) - { - YAML_MAP_BEGIN(yaml) - { - const String *const map = yamlScalarNext(yaml).value; - - yamlEventNextCheck(yaml, yamlEventTypeSeqBegin); - const YamlEvent allowRangeMinVal = yamlScalarNext(yaml); - const YamlEvent allowRangeMaxVal = yamlScalarNext(yaml); - yamlEventNextCheck(yaml, yamlEventTypeSeqEnd); - - MEM_CONTEXT_PRIOR_BEGIN() - { - const BldCfgOptionAllowRangeMapRaw allowRangeMap = - { - .map = strDup(map), - .min = strDup(allowRangeMinVal.value), - .max = strDup(allowRangeMaxVal.value), - }; - - lstAdd(result->mapList, &allowRangeMap); - } - MEM_CONTEXT_PRIOR_END(); - } - YAML_MAP_END(); - } - - yamlEventNextCheck(yaml, yamlEventTypeSeqEnd); - } - } - MEM_CONTEXT_TEMP_END(); - - return result; -} - -// Helper to parse default -static const BldCfgOptionDefault * -bldCfgParseDefaultDup(const BldCfgOptionDefaultRaw *const defaultRaw) -{ - BldCfgOptionDefault *result = NULL; - - if (defaultRaw != NULL) - { - result = memNew(sizeof(BldCfgOptionDefault)); - *result = (BldCfgOptionDefault){0}; - result->value = strDup(defaultRaw->value); - - if (defaultRaw->mapList != NULL) - { - List *const mapList = lstNewP(sizeof(BldCfgOptionDefaultMap), .comparator = lstComparatorStr); - - for (unsigned int mapIdx = 0; mapIdx < lstSize(defaultRaw->mapList); mapIdx++) - { - const BldCfgOptionDefaultMapRaw *const defaultMapRaw = lstGet(defaultRaw->mapList, mapIdx); - const BldCfgOptionDefaultMap defaultMap = - { - .map = strDup(defaultMapRaw->map), - .value = strDup(defaultMapRaw->value), - }; - - lstAdd(mapList, &defaultMap); - } - - result->mapList = mapList; - } - } - - return result; -} - -static const BldCfgOptionDefaultRaw * -bldCfgParseDefault(Yaml *const yaml) -{ - BldCfgOptionDefaultRaw *result = memNew(sizeof(BldCfgOptionDefaultRaw)); - *result = (BldCfgOptionDefaultRaw){0}; - - MEM_CONTEXT_TEMP_BEGIN() - { - // If a scalar then a simple default value - if (yamlEventPeekIs(yaml, yamlEventTypeScalar)) - { - const YamlEvent defaultVal = yamlScalarNext(yaml); - - // If an override to inheritance then set to NULL - if (strEqZ(defaultVal.value, "~")) - result = NULL; - else - { - MEM_CONTEXT_PRIOR_BEGIN() - { - result->value = strDup(defaultVal.value); - } - MEM_CONTEXT_PRIOR_END(); - } - } - // Else a list of per-map defaults - else - { - MEM_CONTEXT_PRIOR_BEGIN() - { - result->mapList = lstNewP(sizeof(BldCfgOptionDefaultMapRaw)); - } - MEM_CONTEXT_PRIOR_END(); - - YAML_SEQ_BEGIN(yaml) - { - YAML_MAP_BEGIN(yaml) - { - const String *const map = yamlScalarNext(yaml).value; - const String *const value = yamlScalarNext(yaml).value; - - MEM_CONTEXT_PRIOR_BEGIN() - { - const BldCfgOptionDefaultMapRaw defaultMap = - { - .map = strDup(map), - .value = strDup(value), - }; - - lstAdd(result->mapList, &defaultMap); - } - MEM_CONTEXT_PRIOR_END(); - } - YAML_MAP_END(); - } - YAML_SEQ_END(); - } - } - MEM_CONTEXT_TEMP_END(); - - return result; -} - -// Helper to parse depend -static const BldCfgOptionDependRaw * -bldCfgParseDepend(Yaml *const yaml, const List *const optList) -{ - const BldCfgOptionDependRaw *result = NULL; - - MEM_CONTEXT_TEMP_BEGIN() - { - // If a map then depend is defined - if (yamlEventPeekIs(yaml, yamlEventTypeMapBegin)) - { - BldCfgOptionDependRaw optDependRaw = {0}; - - YAML_MAP_BEGIN(yaml) - { - const String *const dependDef = yamlScalarNext(yaml).value; - - if (strEqZ(dependDef, "list")) - { - StringList *const valueList = strLstNew(); - - YAML_SEQ_BEGIN(yaml) - { - strLstAdd(valueList, yamlScalarNext(yaml).value); - } - YAML_SEQ_END(); - - optDependRaw.valueList = valueList; - } - else - { - const YamlEvent dependDefVal = yamlScalarNext(yaml); - - if (strEqZ(dependDef, "default")) - optDependRaw.defaultValue = dependDefVal.value; - else if (strEqZ(dependDef, "option")) - optDependRaw.option = dependDefVal.value; - else - THROW_FMT(FormatError, "unknown depend definition '%s'", strZ(dependDef)); - } - } - YAML_MAP_END(); - - MEM_CONTEXT_PRIOR_BEGIN() - { - BldCfgOptionDependRaw *optDepend = memNew(sizeof(BldCfgOptionDependRaw)); - - *optDepend = (BldCfgOptionDependRaw) - { - .option = strDup(optDependRaw.option), - .defaultValue = strDup(optDependRaw.defaultValue), - .valueList = strLstDup(optDependRaw.valueList) - }; - - result = optDepend; - } - MEM_CONTEXT_PRIOR_END(); - } - // Else depend is inherited - else - { - CHECK(AssertError, optList != NULL, "option list is NULL"); - const YamlEvent dependVal = yamlEventNextCheck(yaml, yamlEventTypeScalar); - - const BldCfgOptionRaw *const optInherit = lstFind(optList, &dependVal.value); - - if (optInherit == NULL) - THROW_FMT(FormatError, "dependency inherited from option '%s' before it is defined", strZ(dependVal.value)); - - result = optInherit->depend; - } - } - MEM_CONTEXT_TEMP_END(); - - return result; -} - -// Helper to reconcile depend -static const BldCfgOptionDepend * -bldCfgParseDependReconcile( - const BldCfgOptionRaw *const optRaw, const BldCfgOptionDependRaw *const optDependRaw, const List *const optList) -{ - BldCfgOptionDepend *result = NULL; - - if (optDependRaw != NULL) - { - if (optDependRaw->defaultValue != NULL && !strEq(optRaw->type, OPT_TYPE_BOOLEAN_STR) && - !strEq(optRaw->type, OPT_TYPE_INTEGER_STR)) - { - THROW_FMT(FormatError, "dependency default invalid for non integer/boolean option '%s'", strZ(optRaw->name)); - } - - const BldCfgOption *const optDepend = lstFind(optList, &optDependRaw->option); - - if (optDepend == NULL) - THROW_FMT(FormatError, "dependency on undefined option '%s'", strZ(optDependRaw->option)); - - result = memNew(sizeof(BldCfgOptionDepend)); - *result = (BldCfgOptionDepend) - { - .option = optDepend, - .defaultValue = strDup(optDependRaw->defaultValue), - .valueList = strLstDup(optDependRaw->valueList) - }; - } - - return result; -} - -// Helper to parse deprecate -static List * -bldCfgParseOptionDeprecate(Yaml *const yaml) -{ - List *result = lstNewP(sizeof(BldCfgOptionDeprecateRaw), .comparator = lstComparatorStr); - - MEM_CONTEXT_TEMP_BEGIN() - { - YAML_MAP_BEGIN(yaml) - { - const String *name = yamlScalarNext(yaml).value; - bool indexed = false; - - yamlMapEmpty(yaml); - - // Determine if this deprecation is indexed - const size_t questionPos = (size_t)strChr(name, '?'); - - if (questionPos != (size_t)-1) - { - name = strNewFmt("%s%s", strZ(strSubN(name, 0, questionPos)), strZ(strSub(name, questionPos + 1))); - indexed = true; - } - - // Create final deprecation if it does not already exist - BldCfgOptionDeprecateRaw *deprecate = lstFind(result, &name); - - if (deprecate == NULL) - { - MEM_CONTEXT_PRIOR_BEGIN() - { - lstAdd(result, &(BldCfgOptionDeprecateRaw){.name = strDup(name)}); - } - MEM_CONTEXT_PRIOR_END(); - - deprecate = lstFind(result, &name); - CHECK(AssertError, deprecate != NULL, "deprecate is NULL"); - } - - // Set indexed/unindexed flags - if (indexed) - deprecate->indexed = true; - else - deprecate->unindexed = true; - } - YAML_MAP_END(); - - lstSort(result, sortOrderAsc); - } - MEM_CONTEXT_TEMP_END(); - - return result; -} - -// Helper to reconcile deprecate -static List * -bldCfgParseOptionDeprecateReconcile(const List *const optDeprecateRawList) -{ - List *result = NULL; - - if (optDeprecateRawList != NULL) - { - result = lstNewP(sizeof(BldCfgOptionDeprecate), .comparator = lstComparatorStr); - - for (unsigned int optDeprecateRawIdx = 0; optDeprecateRawIdx < lstSize(optDeprecateRawList); optDeprecateRawIdx++) - { - const BldCfgOptionDeprecateRaw *const optDeprecateRaw = lstGet(optDeprecateRawList, optDeprecateRawIdx); - const BldCfgOptionDeprecate bldCfgOptionDeprecate = - { - .name = strDup(optDeprecateRaw->name), - .indexed = optDeprecateRaw->indexed, - .unindexed = optDeprecateRaw->unindexed, - }; - - lstAdd(result, &bldCfgOptionDeprecate); - } - } - - return result; -} - -// Helper to parse the option command list -static const List * -bldCfgParseOptionCommandList(Yaml *const yaml, const List *const cmdList, const List *const optList) -{ - const List *result = NULL; - - MEM_CONTEXT_TEMP_BEGIN() - { - // If a map then the command list is defined - if (yamlEventPeekIs(yaml, yamlEventTypeMapBegin)) - { - List *const optCmdRawList = lstNewP(sizeof(BldCfgOptionCommandRaw), .comparator = lstComparatorStr); - - YAML_MAP_BEGIN(yaml) - { - const String *const optCmd = yamlScalarNext(yaml).value; - BldCfgOptionCommandRaw optCmdRaw = {.name = optCmd}; - - // Add all commands from an option without attributes - if (strEqZ(optCmd, "+inherit")) - { - CHECK(AssertError, optList != NULL, "option list is NULL"); - - const YamlEvent optInheritVal = yamlScalarNext(yaml); - - const BldCfgOptionRaw *const optInherit = lstFind(optList, &optInheritVal.value); - CHECK(AssertError, optInherit != NULL, "inherited option is NULL"); - - for (unsigned int optCmdRawIdx = 0; optCmdRawIdx < lstSize(optInherit->cmdList); optCmdRawIdx++) - { - const BldCfgOptionCommandRaw *const optCmd = (BldCfgOptionCommandRaw *)lstGet( - optInherit->cmdList, optCmdRawIdx); - lstAdd(optCmdRawList, &(BldCfgOptionCommandRaw){.name = optCmd->name}); - } - } - // Add all commands from a role (or any for all roles) - else if (strEqZ(optCmd, "+role")) - { - const YamlEvent cmdRoleVal = yamlScalarNext(yaml); - - for (unsigned int cmdIdx = 0; cmdIdx < lstSize(cmdList); cmdIdx++) - { - const BldCfgCommand *const cmd = lstGet(cmdList, cmdIdx); - - for (unsigned int cmdRoleIdx = 0; cmdRoleIdx < strLstSize(cmd->roleList); cmdRoleIdx++) - { - const String *const role = strLstGet(cmd->roleList, cmdRoleIdx); - BldCfgOptionCommandRaw optCmdRaw = {.name = cmd->name}; - - if (strEq(cmd->name, CMD_HELP_STR) || strEq(cmd->name, CMD_VERSION_STR)) - continue; - - if ((strEqZ(cmdRoleVal.value, "any") || strEq(role, cmdRoleVal.value)) && - !lstFind(optCmdRawList, &optCmdRaw)) - { - lstAdd(optCmdRawList, &optCmdRaw); - } - } - } - } - // Exclude a command - else if (strEqZ(optCmd, "-command")) - { - const YamlEvent cmdVal = yamlScalarNext(yaml); - - lstRemove(optCmdRawList, &cmdVal.value); - } - // Process the command list - else - { - YAML_MAP_BEGIN(yaml) - { - const String *const optCmdDef = yamlScalarNext(yaml).value; - - if (strEqZ(optCmdDef, "allow-list")) - optCmdRaw.allowList = bldCfgParseAllowList(yaml, NULL); - else if (strEqZ(optCmdDef, "command-role")) - optCmdRaw.roleList = bldCfgParseCommandRole(yaml); - else if (strEqZ(optCmdDef, "depend")) - { - MEM_CONTEXT_BEGIN(lstMemContext(optCmdRawList)) - { - optCmdRaw.depend = bldCfgParseDepend(yaml, optList); - } - MEM_CONTEXT_END(); - } - else if (strEqZ(optCmdDef, "default")) - { - MEM_CONTEXT_BEGIN(lstMemContext(optCmdRawList)) - { - optCmdRaw.defaultValue = bldCfgParseDefault(yaml); - } - MEM_CONTEXT_END(); - } - else - { - const YamlEvent optCmdDefVal = yamlScalarNext(yaml); - - if (strEqZ(optCmdDef, "internal")) - optCmdRaw.internal = varNewBool(yamlBoolParse(optCmdDefVal)); - else if (strEqZ(optCmdDef, "required")) - optCmdRaw.required = varNewBool(yamlBoolParse(optCmdDefVal)); - else if (strEqZ(optCmdDef, "sequence")) - optCmdRaw.sequence = varNewBool(yamlBoolParse(optCmdDefVal)); - else - THROW_FMT(FormatError, "unknown option command definition '%s'", strZ(optCmdDef)); - } - } - YAML_MAP_END(); - - MEM_CONTEXT_BEGIN(lstMemContext(optCmdRawList)) - { - const BldCfgOptionCommandRaw bldCfgOptionCommandRaw = - { - .name = strDup(optCmdRaw.name), - .internal = varDup(optCmdRaw.internal), - .required = varDup(optCmdRaw.required), - .sequence = varDup(optCmdRaw.sequence), - .defaultValue = optCmdRaw.defaultValue, - .depend = optCmdRaw.depend, - .allowList = bldCfgParseAllowListDup(optCmdRaw.allowList), - .roleList = strLstDup(optCmdRaw.roleList), - }; - - lstRemove(optCmdRawList, &bldCfgOptionCommandRaw); - lstAdd(optCmdRawList, &bldCfgOptionCommandRaw); - } - MEM_CONTEXT_END(); - } - } - YAML_MAP_END(); - - lstSort(optCmdRawList, sortOrderAsc); - - result = lstMove(optCmdRawList, memContextPrior()); - } - // Else command list is inherited - else - { - CHECK(AssertError, optList != NULL, "option list is NULL"); - const YamlEvent optCmdVal = yamlEventNextCheck(yaml, yamlEventTypeScalar); - - const BldCfgOptionRaw *const optInherit = lstFind(optList, &optCmdVal.value); - CHECK(AssertError, optInherit != NULL, "inherited option is NULL"); - - result = optInherit->cmdList; - } - } - MEM_CONTEXT_TEMP_END(); - - return result; -} - -static List * -bldCfgParseOptionList(Yaml *const yaml, const List *const cmdList, const List *const optGrpList) -{ - List *const result = lstNewP(sizeof(BldCfgOption), .comparator = lstComparatorStr); - - MEM_CONTEXT_TEMP_BEGIN() - { - List *const optListRaw = lstNewP(sizeof(BldCfgOptionRaw), .comparator = lstComparatorStr); - - // Parse raw - // ------------------------------------------------------------------------------------------------------------------------- - yamlEventNextCheck(yaml, yamlEventTypeScalar); - - YAML_MAP_BEGIN(yaml) - { - const String *const optName = yamlScalarNext(yaml).value; - BldCfgOptionRaw optRaw = {.name = optName, .required = BOOL_TRUE_VAR}; - bool inheritFound = false; - - YAML_MAP_BEGIN(yaml) - { - const String *const optDef = yamlScalarNext(yaml).value; - - if (strEqZ(optDef, "allow-list")) - optRaw.allowList = bldCfgParseAllowList(yaml, optListRaw); - else if (strEqZ(optDef, "allow-range")) - optRaw.allowRange = bldCfgParseAllowRange(yaml); - else if (strEqZ(optDef, "command")) - optRaw.cmdList = bldCfgParseOptionCommandList(yaml, cmdList, optListRaw); - else if (strEqZ(optDef, "command-role")) - optRaw.cmdRoleList = bldCfgParseCommandRole(yaml); - else if (strEqZ(optDef, "default")) - optRaw.defaultValue = bldCfgParseDefault(yaml); - else if (strEqZ(optDef, "depend")) - optRaw.depend = bldCfgParseDepend(yaml, optListRaw); - else if (strEqZ(optDef, "deprecate")) - optRaw.deprecateList = bldCfgParseOptionDeprecate(yaml); - else - { - const YamlEvent optDefVal = yamlScalarNext(yaml); - - if (strEqZ(optDef, "default-type")) - { - if (strEqZ(optDefVal.value, "quote")) - optRaw.defaultType = defaultTypeQuote; - else if (strEqZ(optDefVal.value, "literal")) - optRaw.defaultType = defaultTypeLiteral; - else if (strEqZ(optDefVal.value, "dynamic")) - optRaw.defaultType = defaultTypeDynamic; - else - { - THROW_FMT( - FormatError, "option '%s' has invalid default type '%s'", strZ(optRaw.name), strZ(optDefVal.value)); - } - } - else if (strEqZ(optDef, "group")) - { - optRaw.group = optDefVal.value; - - if (!lstExists(optGrpList, &optRaw.group)) - THROW_FMT(FormatError, "option '%s' has invalid group '%s'", strZ(optRaw.name), strZ(optRaw.group)); - } - else if (strEqZ(optDef, "inherit")) - { - const BldCfgOptionRaw *const optInherit = lstFind(optListRaw, &optDefVal.value); - CHECK(AssertError, optInherit != NULL, "inherited option is NULL"); - - optRaw = *optInherit; - optRaw.name = optName; - - // Deprecations cannot be inherited - optRaw.deprecateList = NULL; - - inheritFound = true; - } - else if (strEqZ(optDef, "internal")) - optRaw.internal = yamlBoolParse(optDefVal); - else if (strEqZ(optDef, "bool-like")) - optRaw.boolLike = yamlBoolParse(optDefVal); - else if (strEqZ(optDef, "beta")) - optRaw.beta = yamlBoolParse(optDefVal); - else if (strEqZ(optDef, "negate")) - optRaw.negate = varNewBool(yamlBoolParse(optDefVal)); - else if (strEqZ(optDef, "sequence")) - optRaw.sequence = varNewBool(yamlBoolParse(optDefVal)); - else if (strEqZ(optDef, "required")) - optRaw.required = varNewBool(yamlBoolParse(optDefVal)); - else if (strEqZ(optDef, "section")) - optRaw.section = optDefVal.value; - else if (strEqZ(optDef, "secure")) - optRaw.secure = yamlBoolParse(optDefVal); - else if (strEqZ(optDef, "type")) - optRaw.type = optDefVal.value; - else - THROW_FMT(FormatError, "unknown option definition '%s'", strZ(optDef)); - } - } - YAML_MAP_END(); - - // Type is required - if (optRaw.type == NULL) - THROW_FMT(FormatError, "option '%s' requires 'type'", strZ(optRaw.name)); - - // Set defaults if not inherited - if (!inheritFound) - { - // Section defaults to command line - if (optRaw.section == NULL) - optRaw.section = SECTION_COMMAND_LINE_STR; - - // Set negate default if not defined - if (optRaw.negate == NULL) - { - optRaw.negate = varNewBool( - (strEq(optRaw.type, OPT_TYPE_BOOLEAN_STR) || optRaw.boolLike) && - !strEq(optRaw.section, SECTION_COMMAND_LINE_STR)); - } - - // Build default command list if not defined - if (optRaw.cmdList == NULL) - { - List *optCmdList = lstNewP(sizeof(BldCfgOptionCommandRaw), .comparator = lstComparatorStr); - - for (unsigned int cmdIdx = 0; cmdIdx < lstSize(cmdList); cmdIdx++) - { - const BldCfgCommand *const cmd = lstGet(cmdList, cmdIdx); - - if (!strEq(cmd->name, CMD_HELP_STR) && !strEq(cmd->name, CMD_VERSION_STR)) - lstAdd(optCmdList, &(BldCfgOptionCommandRaw){.name = cmd->name}); - } - - lstSort(optCmdList, sortOrderAsc); - optRaw.cmdList = optCmdList; - } - } - - // Set reset - optRaw.reset = !strEq(optRaw.section, SECTION_COMMAND_LINE_STR); - - lstAdd(optListRaw, &optRaw); - } - YAML_MAP_END(); - - lstSort(optListRaw, sortOrderAsc); - - // Copy option list to result - // ------------------------------------------------------------------------------------------------------------------------- - for (unsigned int optRawIdx = 0; optRawIdx < lstSize(optListRaw); optRawIdx++) - { - const BldCfgOptionRaw *const optRaw = lstGet(optListRaw, optRawIdx); - - MEM_CONTEXT_BEGIN(lstMemContext(result)) - { - const BldCfgOption bldCfgOption = - { - .name = strDup(optRaw->name), - .type = strDup(optRaw->type), - .section = strDup(optRaw->section), - .boolLike = optRaw->boolLike, - .internal = optRaw->internal, - .beta = optRaw->beta, - .required = varBool(optRaw->required), - .negate = varBool(optRaw->negate), - .reset = optRaw->reset, - .sequence = optRaw->sequence, - .defaultType = optRaw->defaultType, - .defaultValue = bldCfgParseDefaultDup(optRaw->defaultValue), - .group = strDup(optRaw->group), - .secure = optRaw->secure, - .allowList = bldCfgParseAllowListDup(optRaw->allowList), - .allowRange = bldCfgParseAllowRangeDup(optRaw->allowRange), - .deprecateList = bldCfgParseOptionDeprecateReconcile(optRaw->deprecateList), - }; - - lstAdd(result, &bldCfgOption); - } - MEM_CONTEXT_END(); - } - - // Reconcile option list - // ------------------------------------------------------------------------------------------------------------------------- - for (unsigned int optRawIdx = 0; optRawIdx < lstSize(optListRaw); optRawIdx++) - { - const BldCfgOptionRaw *const optRaw = lstGet(optListRaw, optRawIdx); - - // Reconcile option command list roles - List *const cmdOptList = lstNewP(sizeof(BldCfgOptionCommand), .comparator = lstComparatorStr); - - for (unsigned int optCmdRawIdx = 0; optCmdRawIdx < lstSize(optRaw->cmdList); optCmdRawIdx++) - { - BldCfgOptionCommandRaw optCmd = *(BldCfgOptionCommandRaw *)lstGet(optRaw->cmdList, optCmdRawIdx); - - // Lookup command - const BldCfgCommand *const cmd = lstFind(cmdList, &optCmd.name); - - if (cmd == NULL) - { - THROW_FMT( - FormatError, "invalid command '%s' in option '%s' command list", strZ(optCmd.name), strZ(optRaw->name)); - } - - // Default required to option required if not defined - if (optCmd.required == NULL) - optCmd.required = optRaw->required; - - // Default internal to option internal if not defined - if (optCmd.internal == NULL) - optCmd.internal = varNewBool(optRaw->internal); - - // Default sequence to option sequence if not defined - if (optCmd.sequence == NULL) - optCmd.sequence = varNewBool(optRaw->sequence); - - // Default command role list if not defined - if (optCmd.roleList == NULL) - { - // Use option list as default if defined. It will need to be filtered against roles valid for each command - if (optRaw->cmdRoleList != NULL) - { - StringList *const roleList = strLstNew(); - - for (unsigned int cmdRoleIdx = 0; cmdRoleIdx < strLstSize(optRaw->cmdRoleList); cmdRoleIdx++) - { - const String *role = strLstGet(optRaw->cmdRoleList, cmdRoleIdx); - - if (strLstExists(cmd->roleList, role)) - strLstAdd(roleList, role); - } - - optCmd.roleList = roleList; - } - // Else use option command list - else - optCmd.roleList = cmd->roleList; - } - - CHECK(AssertError, optCmd.roleList != NULL, "role list is NULL"); - - MEM_CONTEXT_BEGIN(lstMemContext(cmdOptList)) - { - BldCfgOptionCommand bldCfgOptionCommand = - { - .name = strDup(optCmd.name), - .internal = varBool(optCmd.internal), - .required = varBool(optCmd.required), - .sequence = varBool(optCmd.sequence), - .defaultValue = bldCfgParseDefaultDup(optCmd.defaultValue), - .depend = bldCfgParseDependReconcile(optRaw, optCmd.depend, result), - .allowList = bldCfgParseAllowListDup(optCmd.allowList), - .roleList = strLstDup(optCmd.roleList), - }; - - lstAdd(cmdOptList, &bldCfgOptionCommand); - } - MEM_CONTEXT_END(); - } - - BldCfgOption *const opt = lstGet(result, optRawIdx); - CHECK(AssertError, strEq(opt->name, optRaw->name), "option name does not equal raw option name"); - - MEM_CONTEXT_BEGIN(lstMemContext(result)) - { - opt->cmdList = lstMove(cmdOptList, memContextCurrent()); - opt->depend = bldCfgParseDependReconcile(optRaw, optRaw->depend, result); - } - MEM_CONTEXT_END(); - } - } - MEM_CONTEXT_TEMP_END(); - - return result; -} - -/*********************************************************************************************************************************** -Build option resolve order list -***********************************************************************************************************************************/ -static List * -bldCfgParseOptionResolveList(const List *const optList) -{ - List *const result = lstNewP(sizeof(BldCfgOption *), .comparator = lstComparatorStr); - - // The stanza option will always be resolved first since errors can be confusing when it is missing. That means it must exist - // and cannot have any dependencies. - // ----------------------------------------------------------------------------------------------------------------------------- - const BldCfgOption *const optStanza = lstFind(optList, &OPT_STANZA_STR); - - if (optStanza == NULL) - THROW(FormatError, "option '" OPT_STANZA "' must exist"); - - if (optStanza->depend != NULL) - THROW(FormatError, "option '" OPT_STANZA "' may not depend on other option"); - - for (unsigned int optCmdIdx = 0; optCmdIdx < lstSize(optStanza->cmdList); optCmdIdx++) - { - const BldCfgOptionCommand *const optStanzaCmd = lstGet(optStanza->cmdList, optCmdIdx); - - if (optStanzaCmd->depend != NULL) - THROW_FMT(FormatError, "option '" OPT_STANZA "' command '%s' may not depend on other option", strZ(optStanzaCmd->name)); - } - - // Determine resolve order - // ----------------------------------------------------------------------------------------------------------------------------- - StringList *const optResolveList = strLstNew(); - strLstAdd(optResolveList, OPT_STANZA_STR); - - do - { - // Was at least one option resolved in the loop? - bool resolved = false; - - for (unsigned int optIdx = 0; optIdx < lstSize(optList); optIdx++) - { - const BldCfgOption *const opt = lstGet(optList, optIdx); - - // If the option has already been resolved then continue - if (strLstExists(optResolveList, opt->name)) - continue; - - // If the option dependency has not been resolved then continue - if (opt->depend != NULL && !strLstExists(optResolveList, opt->depend->option->name)) - continue; - - // If the option command dependency has not been resolved then continue - unsigned int optCmdIdx = 0; - - for (; optCmdIdx < lstSize(opt->cmdList); optCmdIdx++) - { - const BldCfgOptionCommand *const optCmd = lstGet(opt->cmdList, optCmdIdx); - - if (optCmd->depend != NULL && !strLstExists(optResolveList, optCmd->depend->option->name)) - break; - } - - if (optCmdIdx < lstSize(opt->cmdList)) - continue; - - // Option dependencies have been resolved - strLstAdd(optResolveList, opt->name); - resolved = true; - } - - // If nothing was resolved in the loop then there may be a circular reference - if (!resolved) - { - // Find the options that were not resolved - StringList *const unresolvedList = strLstNew(); - - for (unsigned int optIdx = 0; optIdx < lstSize(optList); optIdx++) - { - const BldCfgOption *const opt = lstGet(optList, optIdx); - - if (!strLstExists(optResolveList, opt->name)) - strLstAdd(unresolvedList, opt->name); - } - - THROW_FMT( - FormatError, - "unable to resolve dependencies for option(s) '%s'\n" - "HINT: are there circular dependencies?", - strZ(strLstJoin(unresolvedList, ", "))); - } - } - while (strLstSize(optResolveList) != lstSize(optList)); - - // Copy resolved list - for (unsigned int optResolveIdx = 0; optResolveIdx < strLstSize(optResolveList); optResolveIdx++) - { - const String *const optName = strLstGet(optResolveList, optResolveIdx); - const BldCfgOption *const opt = lstFind(optList, &optName); - lstAdd(result, &opt); - } - - return result; -} - -/**********************************************************************************************************************************/ -BldCfg -bldCfgParse(const Storage *const storageRepo) -{ - // Initialize yaml - Yaml *const yaml = yamlNew(storageGetP(storageNewReadP(storageRepo, STRDEF("src/build/config/config.yaml")))); - yamlEventNextCheck(yaml, yamlEventTypeMapBegin); - - // Parse configuration - const List *const cmdList = bldCfgParseCommandList(yaml); - const List *const optGrpList = bldCfgParseOptionGroupList(yaml); - const List *const optList = bldCfgParseOptionList(yaml, cmdList, optGrpList); - const List *const optResolveList = bldCfgParseOptionResolveList(optList); - - return (BldCfg){.cmdList = cmdList, .optGrpList = optGrpList, .optList = optList, .optResolveList = optResolveList}; -} diff --git a/src/build/config/parse.h b/src/build/config/parse.h deleted file mode 100644 index b4fd5240c..000000000 --- a/src/build/config/parse.h +++ /dev/null @@ -1,197 +0,0 @@ -/*********************************************************************************************************************************** -Parse Configuration Yaml -***********************************************************************************************************************************/ -#ifndef BUILD_CONFIG_PARSE_H -#define BUILD_CONFIG_PARSE_H - -#include "storage/storage.h" - -/*********************************************************************************************************************************** -Default types -***********************************************************************************************************************************/ -typedef enum -{ - defaultTypeQuote = 0, - defaultTypeLiteral, - defaultTypeDynamic, -} DefaultType; - -/*********************************************************************************************************************************** -Command role constants -***********************************************************************************************************************************/ -#define CMD_ROLE_ASYNC "async" -STRING_DECLARE(CMD_ROLE_ASYNC_STR); -#define CMD_ROLE_LOCAL "local" -STRING_DECLARE(CMD_ROLE_LOCAL_STR); -#define CMD_ROLE_MAIN "main" -STRING_DECLARE(CMD_ROLE_MAIN_STR); -#define CMD_ROLE_REMOTE "remote" -STRING_DECLARE(CMD_ROLE_REMOTE_STR); - -/*********************************************************************************************************************************** -Command constants -***********************************************************************************************************************************/ -#define CMD_HELP "help" -STRING_DECLARE(CMD_HELP_STR); -#define CMD_VERSION "version" -STRING_DECLARE(CMD_VERSION_STR); - -/*********************************************************************************************************************************** -Option type constants -***********************************************************************************************************************************/ -#define OPT_TYPE_BOOLEAN "boolean" -STRING_DECLARE(OPT_TYPE_BOOLEAN_STR); -#define OPT_TYPE_HASH "hash" -STRING_DECLARE(OPT_TYPE_HASH_STR); -#define OPT_TYPE_INTEGER "integer" -STRING_DECLARE(OPT_TYPE_INTEGER_STR); -#define OPT_TYPE_LIST "list" -STRING_DECLARE(OPT_TYPE_LIST_STR); -#define OPT_TYPE_PATH "path" -STRING_DECLARE(OPT_TYPE_PATH_STR); -#define OPT_TYPE_SIZE "size" -STRING_DECLARE(OPT_TYPE_SIZE_STR); -#define OPT_TYPE_STRING "string" -STRING_DECLARE(OPT_TYPE_STRING_STR); -#define OPT_TYPE_STRING_ID "string-id" -STRING_DECLARE(OPT_TYPE_STRING_ID_STR); -#define OPT_TYPE_TIME "time" -STRING_DECLARE(OPT_TYPE_TIME_STR); - -/*********************************************************************************************************************************** -Option constants -***********************************************************************************************************************************/ -#define OPT_BETA "beta" -STRING_DECLARE(OPT_BETA_STR); -#define OPT_STANZA "stanza" -STRING_DECLARE(OPT_STANZA_STR); - -/*********************************************************************************************************************************** -Section constants -***********************************************************************************************************************************/ -#define SECTION_COMMAND_LINE "command-line" -STRING_DECLARE(SECTION_COMMAND_LINE_STR); -#define SECTION_GLOBAL "global" -STRING_DECLARE(SECTION_GLOBAL_STR); -#define SECTION_STANZA "stanza" -STRING_DECLARE(SECTION_STANZA_STR); - -/*********************************************************************************************************************************** -Types -***********************************************************************************************************************************/ -typedef struct BldCfgCommand -{ - const String *name; // Name - bool internal; // Is the command internal? - bool logFile; // Does the command write automatically to a log file? - const String *logLevelDefault; // Default log level - bool lockRequired; // Is a lock required - bool lockRemoteRequired; // Is a remote lock required? - const String *lockType; // Lock type - bool parameterAllowed; // Are command line parameters allowed? - const StringList *roleList; // Roles valid for the command -} BldCfgCommand; - -typedef struct BldCfgOptionGroup -{ - const String *name; // Name -} BldCfgOptionGroup; - -typedef struct BldCfgOption BldCfgOption; // Forward declaration - -typedef struct BldCfgOptionAllowRangeMap -{ - const String *map; // Map value - const String *min; // Min value - const String *max; // Max value -} BldCfgOptionAllowRangeMap; - -typedef struct BldCfgOptionAllowRange -{ - const String *min; // Min value - const String *max; // Max value - const List *mapList; // List of default mappings -} BldCfgOptionAllowRange; - -typedef struct BldCfgOptionDefaultMap -{ - const String *map; // Map value - const String *value; // Default value -} BldCfgOptionDefaultMap; - -typedef struct BldCfgOptionDefault -{ - const String *value; // Default value - const List *mapList; // List of default mappings -} BldCfgOptionDefault; - -typedef struct BldCfgOptionDepend -{ - const BldCfgOption *option; // Option dependency is on - const String *defaultValue; // Default value, if any, when dependency is not resolved - const StringList *valueList; // Allowed value list -} BldCfgOptionDepend; - -typedef struct BldCfgOptionDeprecate -{ - const String *name; // Deprecated option name - bool indexed; // Can the deprecation be indexed? - bool unindexed; // Can the deprecation be unindexed? -} BldCfgOptionDeprecate; - -typedef struct BldCfgOptionCommand -{ - const String *name; // Name - bool internal; // Is the option internal? - bool required; // Is the option required? - bool sequence; // Sequence added to StringId? - const BldCfgOptionDefault *defaultValue; // Default value, if any - const BldCfgOptionDepend *depend; // Dependency, if any - const List *allowList; // Allowed value list - const StringList *roleList; // Roles valid for the command -} BldCfgOptionCommand; - -typedef struct BldCfgOptionValue -{ - const String *value; // Option value - const String *condition; // Is the option conditionally compiled? -} BldCfgOptionValue; - -struct BldCfgOption -{ - const String *name; // Name - const String *type; // Option type, e.g. integer - const String *section; // Option section, i.e. stanza or global - bool boolLike; // Option accepts y/n and can be treated as bool? - bool internal; // Is the option internal? - bool beta; // Is the option beta? - bool required; // Is the option required? - bool negate; // Can the option be negated? - bool reset; // Can the option be reset? - bool sequence; // Sequence added to StringId? - DefaultType defaultType; // Type of default - const BldCfgOptionDefault *defaultValue; // Default value, if any - const String *group; // Option group, if any - bool secure; // Does the option contain a secret? - const BldCfgOptionDepend *depend; // Dependency, if any - const List *allowList; // Allowed value list - const BldCfgOptionAllowRange *allowRange; // Allow range, if any - const List *cmdList; // Command override list - const List *deprecateList; // List of option deprecations -}; - -typedef struct BldCfg -{ - const List *cmdList; // Command list - const List *optGrpList; // Option group list - const List *optList; // Option list - const List *optResolveList; // Option list in resolved dependency order -} BldCfg; - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Parse config.yaml -BldCfg bldCfgParse(const Storage *const storageRepo); - -#endif diff --git a/src/build/config/render.c b/src/build/config/render.c deleted file mode 100644 index e79a162bf..000000000 --- a/src/build/config/render.c +++ /dev/null @@ -1,1523 +0,0 @@ -/*********************************************************************************************************************************** -Render Configuration Data -***********************************************************************************************************************************/ -#include - -#include "common/log.h" -#include "common/type/convert.h" -#include "config/common.h" -#include "storage/posix/storage.h" - -#include "build/common/render.h" -#include "build/config/render.h" - -/*********************************************************************************************************************************** -Option type constants -***********************************************************************************************************************************/ -#define CFGDEF_TYPE_STRING "string" - -/*********************************************************************************************************************************** -Build constant from a string -***********************************************************************************************************************************/ -static String * -bldConst(const char *const prefix, const String *const value) -{ - return strUpper(strReplaceChr(strNewFmt("%s_%s", prefix, strZ(value)), '-', '_')); -} - -/*********************************************************************************************************************************** -Build enum from a string -***********************************************************************************************************************************/ -// Build command enum from a string -static String * -bldEnumCmd(const String *const value) -{ - return bldEnum("cfgCmd", value); -} - -// Build option group enum from a string -static String * -bldEnumOptGrp(const String *const value) -{ - return bldEnum("cfgOptGrp", value); -} - -// Build option enum from a string -static String * -bldEnumOpt(const String *const value) -{ - return bldEnum("cfgOpt", value); -} - -/*********************************************************************************************************************************** -Render config.auto.h -***********************************************************************************************************************************/ -#define CONFIG_MODULE "config" -#define CONFIG_AUTO_COMMENT "Command and Option Configuration" - -static void -bldCfgRenderConfigAutoH(const Storage *const storageRepo, const BldCfg bldCfg) -{ - String *config = strCatFmt( - strNew(), - "%s" - "#ifndef CONFIG_CONFIG_AUTO_H\n" - "#define CONFIG_CONFIG_AUTO_H\n", - strZ(bldHeader(CONFIG_MODULE, CONFIG_AUTO_COMMENT))); - - // Command constants - // ----------------------------------------------------------------------------------------------------------------------------- - strCatZ( - config, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Command constants\n" - COMMENT_BLOCK_END "\n"); - - for (unsigned int cmdIdx = 0; cmdIdx < lstSize(bldCfg.cmdList); cmdIdx++) - { - const BldCfgCommand *const cmd = lstGet(bldCfg.cmdList, cmdIdx); - - strCatFmt(config, "%s\n", strZ(bldDefineRender(bldConst("CFGCMD", cmd->name), strNewFmt("\"%s\"", strZ(cmd->name))))); - } - - strCatFmt(config, "\n%s\n", strZ(bldDefineRender(STRDEF("CFG_COMMAND_TOTAL"), strNewFmt("%u", lstSize(bldCfg.cmdList))))); - - // Option group constants - // ----------------------------------------------------------------------------------------------------------------------------- - strCatZ( - config, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Option group constants\n" - COMMENT_BLOCK_END "\n"); - - strCatFmt(config, "%s\n", strZ(bldDefineRender(STRDEF("CFG_OPTION_GROUP_TOTAL"), strNewFmt("%u", lstSize(bldCfg.optGrpList))))); - - // Option constants - // ----------------------------------------------------------------------------------------------------------------------------- - strCatZ( - config, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Option constants\n" - COMMENT_BLOCK_END "\n"); - - for (unsigned int optIdx = 0; optIdx < lstSize(bldCfg.optList); optIdx++) - { - const BldCfgOption *const opt = lstGet(bldCfg.optList, optIdx); - - if (opt->group == NULL) - strCatFmt(config, "%s\n", strZ(bldDefineRender(bldConst("CFGOPT", opt->name), strNewFmt("\"%s\"", strZ(opt->name))))); - } - - strCatFmt(config, "\n%s\n", strZ(bldDefineRender(STRDEF("CFG_OPTION_TOTAL"), strNewFmt("%u", lstSize(bldCfg.optList))))); - - // Option value constants - // ----------------------------------------------------------------------------------------------------------------------------- - bool lf = false; - - strCatZ( - config, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Option value constants\n" - COMMENT_BLOCK_END "\n"); - - for (unsigned int optIdx = 0; optIdx < lstSize(bldCfg.optList); optIdx++) - { - const BldCfgOption *const opt = lstGet(bldCfg.optList, optIdx); - - if (strEq(opt->type, OPT_TYPE_STRING_ID_STR)) - { - StringList *const groupList = strLstNew(); - List *const groupAllowList = lstNewP(sizeof(StringList *)); - StringList *const optionList = strLstNew(); - - strLstAddZ(groupList, "@"); - lstAdd(groupAllowList, &optionList); - - if (opt->allowList != NULL) - { - for (unsigned int allowListIdx = 0; allowListIdx < lstSize(opt->allowList); allowListIdx++) - strLstAddIfMissing(optionList, ((const BldCfgOptionValue *)lstGet(opt->allowList, allowListIdx))->value); - } - - for (unsigned int optCmdListIdx = 0; optCmdListIdx < lstSize(opt->cmdList); optCmdListIdx++) - { - BldCfgOptionCommand *optCmd = lstGet(opt->cmdList, optCmdListIdx); - - if (optCmd->allowList != NULL) - { - StringList *const cmdList = strLstNew(); - - for (unsigned int allowListIdx = 0; allowListIdx < lstSize(optCmd->allowList); allowListIdx++) - { - const String *const value = ((const BldCfgOptionValue *)lstGet(optCmd->allowList, allowListIdx))->value; - - if (optCmd->sequence) - strLstAdd(cmdList, value); - else - strLstAddIfMissing(optionList, value); - } - - strLstAdd(groupList, strUpper(strDup(optCmd->name))); - lstAdd(groupAllowList, &cmdList); - } - } - - if (!opt->sequence) - strLstSort(optionList, sortOrderAsc); - - for (unsigned int groupIdx = 0; groupIdx < strLstSize(groupList); groupIdx++) - { - const String *const group = - strEqZ(strLstGet(groupList, groupIdx), "@") ? - EMPTY_STR : strNewFmt("%s_", strZ(strLstGet(groupList, groupIdx))); - const StringList *const allowList = *(StringList **)lstGet(groupAllowList, groupIdx); - const bool sequence = (groupIdx == 0 && opt->sequence) || (groupIdx != 0 && !strLstEmpty(allowList)); - - if (lf && !strLstEmpty(allowList)) - strCatChr(config, '\n'); - - for (unsigned int allowListIdx = 0; allowListIdx < strLstSize(allowList); allowListIdx++) - { - const String *const allowListItem = strLstGet(allowList, allowListIdx); - const String *const constPrefix = strUpper( - strReplaceChr(strNewFmt("CFGOPTVAL_%s%s_%s", strZ(group), strZ(opt->name), strZ(allowListItem)), '-', '_')); - - // Render sequence - if (sequence) - strCatFmt(config, "%s\n", strZ(bldDefineRender(constPrefix, strNewFmt("%u", allowListIdx)))); - - // Render StringId - strCatFmt( - config, "%s\n", - strZ( - bldDefineRender( - strNewFmt("%s%s", strZ(constPrefix), sequence ? "_STRID" : ""), - bldStrIdSeq(strZ(allowListItem), sequence ? allowListIdx : STRING_ID_SEQ_NONE)))); - - // Render Z - strCatFmt( - config, "%s\n", - strZ(bldDefineRender(strNewFmt("%s_Z", strZ(constPrefix)), strNewFmt("\"%s\"", strZ(allowListItem))))); - - lf = true; - } - } - } - } - - // Command enum - // ----------------------------------------------------------------------------------------------------------------------------- - strCatZ( - config, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Command enum\n" - COMMENT_BLOCK_END "\n" - "typedef enum\n" - "{\n"); - - for (unsigned int cmdIdx = 0; cmdIdx < lstSize(bldCfg.cmdList); cmdIdx++) - { - const BldCfgCommand *const cmd = lstGet(bldCfg.cmdList, cmdIdx); - - strCatFmt(config, " %s,\n", strZ(bldEnumCmd(cmd->name))); - } - - strCatZ( - config, - "} ConfigCommand;\n"); - - // Option group enum - // ----------------------------------------------------------------------------------------------------------------------------- - strCatZ( - config, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Option group enum\n" - COMMENT_BLOCK_END "\n" - "typedef enum\n" - "{\n"); - - for (unsigned int optGrpIdx = 0; optGrpIdx < lstSize(bldCfg.optGrpList); optGrpIdx++) - { - const BldCfgOptionGroup *const optGrp = lstGet(bldCfg.optGrpList, optGrpIdx); - - strCatFmt(config, " %s,\n", strZ(bldEnumOptGrp(optGrp->name))); - } - - strCatZ( - config, - "} ConfigOptionGroup;\n"); - - // Option enum - // ----------------------------------------------------------------------------------------------------------------------------- - strCatZ( - config, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Option enum\n" - COMMENT_BLOCK_END "\n" - "typedef enum\n" - "{\n"); - - for (unsigned int optIdx = 0; optIdx < lstSize(bldCfg.optList); optIdx++) - { - const BldCfgOptionGroup *const opt = lstGet(bldCfg.optList, optIdx); - - strCatFmt(config, " %s,\n", strZ(bldEnumOpt(opt->name))); - } - - strCatZ( - config, - "} ConfigOption;\n"); - - // End and save - strCatZ( - config, - "\n" - "#endif\n"); - - bldPut(storageRepo, "src/config/config.auto.h", BUFSTR(config)); -} - -/*********************************************************************************************************************************** -Render parse.auto.c.inc -***********************************************************************************************************************************/ -#define PARSE_AUTO_COMMENT "Config Parse Rules" - -typedef struct BldCfgRenderOptionDeprecate -{ - const String *const name; // Deprecated option name - const BldCfgOption *const option; // Option - const bool indexed; // Can the deprecation be indexed? - const bool unindexed; // Can the deprecation be unindexed? -} BldCfgRenderOptionDeprecate; - -static void -bldCfgRenderLf(String *const config, const bool lf) -{ - if (lf) - strCatZ(config, "\n"); -} - -static String * -bldCfgRenderLabel(const String *const config, const bool label, const String *const labelText) -{ - const StringList *const lines = strLstNewSplitZ(config, "\n"); - String *const result = strNew(); - - const String *const labelComment = strNewFmt("// %s", strZ(labelText)); - - for (unsigned int lineIdx = 0; lineIdx < strLstSize(lines); lineIdx++) - { - const String *const line = strLstGet(lines, lineIdx); - - if (lineIdx != 0) - strCatChr(result, '\n'); - - if (!label || strSize(line) + 1 + strSize(labelComment) > 132) - strCat(result, line); - else - strCatFmt(result, "%s%*s", strZ(line), (int)(132 - strSize(line)), strZ(labelComment)); - } - - return result; -} - -// Helper to get var-128 encoding size -static size_t -bldCfgRenderVar128Size(uint64_t value) -{ - size_t result = 1; - - while (value >= 0x80) - { - value >>= 7; - result++; - } - - return result; -} - -// Helper to render enum string -static String * -bldCfgRenderEnumStr(const String *const source) -{ - String *const result = strNew(); - bool priorSpecial = false; - - for (unsigned int sourceIdx = 0; sourceIdx < strSize(source); sourceIdx++) - { - switch (strZ(source)[sourceIdx]) - { - case '"': - case '/': - case ' ': - case '.': - case '-': - { - if (sourceIdx != 0 && !priorSpecial) - strCatChr(result, '_'); - - switch (strZ(source)[sourceIdx]) - { - case '"': - strCatZ(result, "QT"); - break; - - case '/': - strCatZ(result, "FS"); - break; - - case ' ': - strCatZ(result, "SP"); - break; - - case '.': - strCatZ(result, "DT"); - break; - - case '-': - strCatZ(result, "DS"); - break; - } - - if (sourceIdx != strSize(source) - 1) - strCatChr(result, '_'); - - priorSpecial = true; - break; - } - - default: - strCatChr(result, strZ(source)[sourceIdx]); - priorSpecial = false; - break; - } - } - - return result; -} - -// Helper to render scalars -static String * -bldCfgRenderScalar(const String *const scalar, const String *const optType) -{ - if (strEq(optType, OPT_TYPE_STRING_ID_STR)) - return strNewFmt("PARSE_RULE_VAL_STRID(%s)", strZ(bldCfgRenderEnumStr(scalar))); - - if (strEq(optType, OPT_TYPE_STRING_STR)) - return strNewFmt("PARSE_RULE_VAL_STR(%s)", strZ(bldCfgRenderEnumStr(scalar))); - - if (strEq(optType, OPT_TYPE_BOOLEAN_STR)) - return strNewFmt("PARSE_RULE_VAL_BOOL_%s", strEqZ(scalar, "true") ? "TRUE" : "FALSE"); - - if (strEq(optType, OPT_TYPE_INTEGER_STR)) - return strNewFmt("PARSE_RULE_VAL_INT(%s)", strZ(bldCfgRenderEnumStr(scalar))); - - if (strEq(optType, OPT_TYPE_SIZE_STR)) - return strNewFmt("PARSE_RULE_VAL_SIZE(%s)", strZ(scalar)); - - CHECK_FMT(AssertError, strEq(optType, OPT_TYPE_TIME_STR), "invalid type '%s'", strZ(optType)); - - return strNewFmt("PARSE_RULE_VAL_TIME(%s)", strZ(scalar)); - - // WHERE IS THE 1 TIME COMING FROM? -} - -// Helper to render validity -static String * -bldCfgRenderValid(const String *const type, const BldCfgOptionDepend *const depend) -{ - ASSERT(depend != NULL); - - String *const result = strNew(); - - strCatZ( - result, - " PARSE_RULE_OPTIONAL_DEPEND\n" - " (\n"); - - if (depend->defaultValue != NULL) - { - strCatFmt( - result, - " PARSE_RULE_OPTIONAL_DEPEND_DEFAULT(%s),\n", strZ(bldCfgRenderScalar(depend->defaultValue, type))); - } - - strCatFmt( - result, - " PARSE_RULE_VAL_OPT(%s),\n", - strZ(bldEnum("", depend->option->name))); - - if (depend->valueList != NULL) - { - for (unsigned int valueIdx = 0; valueIdx < strLstSize(depend->valueList); valueIdx++) - { - strCatFmt( - result, - " %s,\n", - strZ( - bldCfgRenderScalar( - strLstGet(depend->valueList, valueIdx), - strEq(depend->option->type, OPT_TYPE_STRING_STR) ? OPT_TYPE_STRING_ID_STR : depend->option->type))); - } - } - - strCatZ( - result, - " )"); - - return result; -} - -// Helper to render allow range -static String * -bldCfgRenderAllowRange(const BldCfgOptionAllowRange *const allowRange, const String *const optType) -{ - ASSERT((allowRange->min != NULL && allowRange->min != NULL) || (allowRange->min == NULL && allowRange->min == NULL)); - ASSERT((allowRange->mapList == NULL && allowRange->min != NULL) || (allowRange->mapList != NULL && allowRange->min == NULL)); - ASSERT(optType != NULL); - - String *const result = strCatZ( - strNew(), - " PARSE_RULE_OPTIONAL_ALLOW_RANGE\n" - " (\n"); - - if (allowRange->mapList != NULL) - { - strCatFmt( - result, - " PARSE_RULE_OPTIONAL_ALLOW_RANGE_MAP\n" - " (\n"); - - for (unsigned int mapIdx = 0; mapIdx < lstSize(allowRange->mapList); mapIdx++) - { - const BldCfgOptionAllowRangeMap *const allowRangeMap = lstGet(allowRange->mapList, mapIdx); - - strCatFmt( - result, - " %s,\n" - " %s,\n" - " %s,\n", - strZ(bldCfgRenderScalar(allowRangeMap->map, OPT_TYPE_STRING_ID_STR)), - strZ(bldCfgRenderScalar(allowRangeMap->min, optType)), - strZ(bldCfgRenderScalar(allowRangeMap->max, optType))); - } - - strCatZ( - result, - " ),\n"); - } - else - { - strCatFmt( - result, - " %s,\n" - " %s,\n", - strZ(bldCfgRenderScalar(allowRange->min, optType)), - strZ(bldCfgRenderScalar(allowRange->max, optType))); - } - - strCatZ( - result, - " )"); - - return result; -} - -// Helper to render allow list -static String * -bldCfgRenderAllowList(const List *const allowList, const String *const optType) -{ - ASSERT(allowList != NULL); - ASSERT(optType != NULL); - - String *const result = strNew(); - - strCatZ( - result, - " PARSE_RULE_OPTIONAL_ALLOW_LIST\n" - " (\n"); - - for (unsigned int allowIdx = 0; allowIdx < lstSize(allowList); allowIdx++) - { - const BldCfgOptionValue *const allow = lstGet(allowList, allowIdx); - const String *const value = strEq(optType, OPT_TYPE_STRING_STR) ? strNewFmt("\"%s\"", strZ(allow->value)) : allow->value; - - strCatFmt(result, " %s,\n", strZ(bldCfgRenderScalar(value, optType))); - - if (allow->condition != NULL) - { - strCatFmt( - result, - "#ifndef %s\n" - " PARSE_RULE_BOOL_FALSE,\n" - "#endif\n", - strZ(allow->condition)); - } - } - - strCatZ(result, " )"); - - return result; -} - -// Helper to render default -static void -bldCfgRenderDefaultValue( - String *const result, const String *const defaultValue, const DefaultType defaultType, const String *const optType, - const bool indent) -{ - ASSERT(defaultValue != NULL); - ASSERT(optType != NULL); - - if (indent) - strCatZ(result, " "); - - if (!strEq(optType, OPT_TYPE_STRING_STR) && !strEq(optType, OPT_TYPE_PATH_STR)) - { - strCatFmt(result, " %s,\n", strZ(bldCfgRenderScalar(defaultValue, optType))); - } - else - { - strCatFmt( - result, - " %s,\n", - strZ( - bldCfgRenderScalar( - strNewFmt( - "%s%s%s", defaultType == defaultTypeLiteral ? "" : "\"", strZ(defaultValue), - defaultType == defaultTypeLiteral ? "" : "\""), OPT_TYPE_STRING_STR))); - } -} - -static String * -bldCfgRenderDefault( - const BldCfgOptionDefault *const defaultValue, const DefaultType defaultType, const String *const optType, - const bool sequence, const List *const allowList) -{ - ASSERT(defaultValue != NULL); - ASSERT(optType != NULL); - - String *const result = strNew(); - - strCatZ( - result, - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n"); - - if (defaultType == defaultTypeDynamic) - { - strCatFmt(result, " PARSE_RULE_DEFAULT_DYNAMIC(%s),\n", strZ(bldEnum("", defaultValue->value))); - } - else if (defaultValue->value != NULL) - { - bldCfgRenderDefaultValue(result, defaultValue->value, defaultType, optType, false); - - if (sequence) - { - unsigned int allowIdx = lstFindIdx(allowList, &defaultValue->value); - CHECK_FMT( - FormatError, allowIdx != LIST_NOT_FOUND, "unable to find default '%s' in allow list", strZ(defaultValue->value)); - - if (allowIdx != 0) - strCatFmt(result, " PARSE_RULE_VAL_SEQ(%u),\n", allowIdx); - } - } - else - { - ASSERT(defaultValue->mapList != NULL); - ASSERT(!sequence); - - strCatZ( - result, - " PARSE_RULE_OPTIONAL_DEFAULT_MAP\n" - " (\n"); - - for (unsigned int mapIdx = 0; mapIdx < lstSize(defaultValue->mapList); mapIdx++) - { - const BldCfgOptionDefaultMap *const map = lstGet(defaultValue->mapList, mapIdx); - - strCatFmt(result, " %s,\n", strZ(bldCfgRenderScalar(map->map, OPT_TYPE_STRING_ID_STR))); - - bldCfgRenderDefaultValue(result, map->value, defaultType, optType, true); - } - - strCatZ(result, " ),\n"); - } - - strCatZ(result, " )"); - - return result; -} - -// Helper to add values to value lists -#define BLD_CFG_RENDER_VALUE_MAP_MAX UINT8_MAX - -static void -bldCfgRenderValueAdd(const String *optType, const DefaultType defaultType, const String *const value, KeyValue *const ruleValMap) -{ - ASSERT(!strEq(optType, OPT_TYPE_BOOLEAN_STR) && !strEq(optType, OPT_TYPE_HASH_STR) && !strEq(optType, OPT_TYPE_LIST_STR)); - - // Dynamic defaults are generated at runtime and do not need to be added to a static values list - if (defaultType == defaultTypeDynamic) - return; - - // Remap path to string - if (strEq(optType, OPT_TYPE_PATH_STR)) - optType = OPT_TYPE_STRING_STR; - - // Generate string value - const String *const valueStr = strNewFmt( - "%s%s%s", defaultType == defaultTypeLiteral ? "" : "\"", strZ(value), defaultType == defaultTypeLiteral ? "" : "\""); - - // Add values other than strings - if (!strEq(optType, OPT_TYPE_STRING_STR)) - { - if (kvGet(ruleValMap, VARSTR(optType)) == NULL) - kvPutKv(ruleValMap, VARSTR(optType)); - - kvPut(varKv(kvGet(ruleValMap, VARSTR(optType))), VARSTR(value), VARSTR(valueStr)); - } - - // Always add to string list - if (kvGet(ruleValMap, VARSTR(OPT_TYPE_STRING_STR)) == NULL) - kvPutKv(ruleValMap, VARSTR(OPT_TYPE_STRING_STR)); - - kvPut(varKv(kvGet(ruleValMap, VARSTR(OPT_TYPE_STRING_STR))), VARSTR(valueStr), NULL); -} - -// Helper to render value lists -static int -bldCfgRenderComparatorInt(const void *const int64Ptr1, const void *const int64Ptr2) -{ - return LST_COMPARATOR_CMP( - cvtZToInt64(strZ(*(const String *const *)int64Ptr1)), cvtZToInt64(strZ(*(const String *const *)int64Ptr2))); -} - -static int -bldCfgRenderComparatorSize(const void *const sizePtr1, const void *const sizePtr2) -{ - return LST_COMPARATOR_CMP(cfgParseSize(*(const String *const *)sizePtr1), cfgParseSize(*(const String *const *)sizePtr2)); -} - -static int -bldCfgRenderComparatorTime(const void *const timePtr1, const void *const timePtr2) -{ - return LST_COMPARATOR_CMP(cfgParseTime(*(const String *const *)timePtr1), cfgParseTime(*(const String *const *)timePtr2)); -} - -static String * -bldCfgRenderValueRender( - const String *const optType, const KeyValue *const ruleValAllMap, const bool label, const char *const type, - const char *const caption, const char *const abbr, const char *const macro, const char *const comment) -{ - String *const result = strNew(); - const KeyValue *const ruleValMap = varKv(kvGet(ruleValAllMap, VARSTR(optType))); - StringList *const ruleValList = strLstNewVarLst(kvKeyList(ruleValMap)); - ASSERT(!strEq(optType, OPT_TYPE_STRING_STR) || strLstSize(ruleValList) <= BLD_CFG_RENDER_VALUE_MAP_MAX); - - // Sort list - if (strEq(optType, OPT_TYPE_INTEGER_STR)) - lstComparatorSet((List *)ruleValList, bldCfgRenderComparatorInt); - else if (strEq(optType, OPT_TYPE_SIZE_STR)) - lstComparatorSet((List *)ruleValList, bldCfgRenderComparatorSize); - else if (strEq(optType, OPT_TYPE_TIME_STR)) - lstComparatorSet((List *)ruleValList, bldCfgRenderComparatorTime); - - strLstSort(ruleValList, sortOrderAsc); - - // Render value macro - strCatFmt( - result, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Rule %ss\n" - COMMENT_BLOCK_END "\n", - caption); - - strCatFmt( - result, "%s\n", - strZ( - bldDefineRender( - strNewFmt("PARSE_RULE_VAL_%s(value)", macro), - strNewFmt( - "PARSE_RULE_U32_%zu(parseRuleVal%s##value)", bldCfgRenderVar128Size(strLstSize(ruleValList) - 1), abbr)))); - - // Add value macro for StringId - if (strEq(optType, OPT_TYPE_STRING_STR)) - { - strCatFmt( - result, "%s\n", - strZ(bldDefineRender(STRDEF("PARSE_RULE_VAL_STRID(value)"), STRDEF("PARSE_RULE_VAL_STR(QT_##value##_QT)")))); - } - - // Render values - strCatFmt( - result, - "\n" - "static const %s parseRuleValue%s[] =\n" - "{\n", - type, abbr); - - String *const resultVal = strNew(); - - for (unsigned int ruleValIdx = 0; ruleValIdx < strLstSize(ruleValList); ruleValIdx++) - { - bldCfgRenderLf(resultVal, ruleValIdx != 0); - - if (strEq(optType, OPT_TYPE_STRING_STR)) - strCatFmt(resultVal, " PARSE_RULE_STRPUB(%s),", strZ(strLstGet(ruleValList, ruleValIdx))); - else if (strEq(optType, OPT_TYPE_INTEGER_STR)) - strCatFmt(resultVal, " %s,", strZ(strTrim(strLstGet(ruleValList, ruleValIdx)))); - else if (strEq(optType, OPT_TYPE_SIZE_STR)) - strCatFmt(resultVal, " %" PRId64 ",", cfgParseSize(strLstGet(ruleValList, ruleValIdx))); - else - strCatFmt(resultVal, " %" PRId64 ",", cfgParseTime(strLstGet(ruleValList, ruleValIdx))); - } - - strCat(result, bldCfgRenderLabel(resultVal, label, STR(comment))); - strCatZ(result, "\n};\n"); - - // Render value to string map - if (strEq(optType, OPT_TYPE_INTEGER_STR) || strEq(optType, OPT_TYPE_SIZE_STR) || strEq(optType, OPT_TYPE_TIME_STR)) - { - strCatFmt( - result, - "\n" - "static const uint8_t parseRuleValue%sStrMap[] =\n" - "{\n", - abbr); - - String *const resultMap = strNew(); - - for (unsigned int ruleValIdx = 0; ruleValIdx < strLstSize(ruleValList); ruleValIdx++) - { - const Variant *const ruleStrId = kvGet(ruleValMap, VARSTR(strLstGet(ruleValList, ruleValIdx))); - ASSERT(ruleStrId != NULL); - - bldCfgRenderLf(resultMap, ruleValIdx != 0); - strCatFmt(resultMap, " %s,", zNewFmt("parseRuleValStr%s", strZ(bldCfgRenderEnumStr(varStr(ruleStrId))))); - } - - strCat(result, bldCfgRenderLabel(resultMap, label, strNewFmt("%s/strmap", comment))); - strCatZ(result, "\n};\n"); - } - - // Render value enum - strCatZ( - result, - "\n" - "typedef enum\n" - "{\n"); - - String *const resultEnum = strNew(); - - for (unsigned int ruleValIdx = 0; ruleValIdx < strLstSize(ruleValList); ruleValIdx++) - { - bldCfgRenderLf(resultEnum, ruleValIdx != 0); - strCatFmt(resultEnum, " parseRuleVal%s", abbr); - - if (strEq(optType, OPT_TYPE_STRING_STR) || strEq(optType, OPT_TYPE_INTEGER_STR)) - strCatFmt(resultEnum, "%s,", strZ(bldCfgRenderEnumStr(strLstGet(ruleValList, ruleValIdx)))); - else - strCatFmt(resultEnum, "%s,", strZ(bldEnum("", strLstGet(ruleValList, ruleValIdx)))); - } - - strCat(result, bldCfgRenderLabel(resultEnum, label, strNewFmt("%s/enum", comment))); - strCatFmt(result, "\n} ParseRuleValue%s;\n", abbr); - - return result; -} - -static void -bldCfgRenderParseAutoC(const Storage *const storageRepo, const BldCfg bldCfg, const bool label) -{ - String *const config = strNew(); - KeyValue *const ruleValMap = kvNew(); - - // Command parse rules - // ----------------------------------------------------------------------------------------------------------------------------- - strCatZ( - config, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Command parse data\n" - COMMENT_BLOCK_END "\n"); - - strCatFmt( - config, "%s\n", - strZ( - bldDefineRender( - STRDEF("PARSE_RULE_VAL_CMD(value)"), - strNewFmt("PARSE_RULE_U32_%zu(cfgCmd##value)", bldCfgRenderVar128Size(lstSize(bldCfg.cmdList) - 1))))); - - strCatZ( - config, - "\n" - "static const ParseRuleCommand parseRuleCommand[CFG_COMMAND_TOTAL] =\n" - "{\n"); - - for (unsigned int cmdIdx = 0; cmdIdx < lstSize(bldCfg.cmdList); cmdIdx++) - { - const BldCfgCommand *const cmd = lstGet(bldCfg.cmdList, cmdIdx); - String *const configCmd = strNew(); - - if (cmdIdx != 0) - strCatZ(config, COMMENT_SEPARATOR "\n"); - - strCatFmt( - configCmd, - " PARSE_RULE_COMMAND\n" - " (\n" - " PARSE_RULE_COMMAND_NAME(\"%s\"),\n", - strZ(cmd->name)); - - if (cmd->internal) - strCatZ(configCmd, " PARSE_RULE_COMMAND_INTERNAL(true)\n"); - - if (cmd->lockRequired) - strCatZ(configCmd, " PARSE_RULE_COMMAND_LOCK_REQUIRED(true),\n"); - - if (cmd->lockRemoteRequired) - strCatZ(configCmd, " PARSE_RULE_COMMAND_LOCK_REMOTE_REQUIRED(true),\n"); - - strCatFmt(configCmd, " PARSE_RULE_COMMAND_LOCK_TYPE(%s),\n", strZ(bldEnum("", cmd->lockType))); - - if (cmd->logFile) - strCatZ(configCmd, " PARSE_RULE_COMMAND_LOG_FILE(true),\n"); - - strCatFmt( - configCmd, " PARSE_RULE_COMMAND_LOG_LEVEL_DEFAULT(%s),\n", strZ(bldEnum("", cmd->logLevelDefault))); - - if (cmd->parameterAllowed) - strCatZ(configCmd, " PARSE_RULE_COMMAND_PARAMETER_ALLOWED(true),\n"); - - strCatZ( - configCmd, - "\n" - " PARSE_RULE_COMMAND_ROLE_VALID_LIST\n" - " (\n"); - - for (unsigned int cmdRoleIdx = 0; cmdRoleIdx < strLstSize(cmd->roleList); cmdRoleIdx++) - { - strCatFmt( - configCmd, - " PARSE_RULE_COMMAND_ROLE(%s)\n", - strZ(bldEnum("", strLstGet(cmd->roleList, cmdRoleIdx)))); - } - - strCatZ( - configCmd, - " ),\n" - " ),"); - - strCat(config, bldCfgRenderLabel(configCmd, label, strNewFmt("cmd/%s", strZ(cmd->name)))); - strCatChr(config, '\n'); - } - - strCatZ( - config, - "};\n"); - - // Option group rules - // ----------------------------------------------------------------------------------------------------------------------------- - strCatZ( - config, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Option group parse data\n" - COMMENT_BLOCK_END "\n" - "static const ParseRuleOptionGroup parseRuleOptionGroup[CFG_OPTION_GROUP_TOTAL] =\n" - "{\n"); - - for (unsigned int optGrpIdx = 0; optGrpIdx < lstSize(bldCfg.optGrpList); optGrpIdx++) - { - const BldCfgOptionGroup *const optGrp = lstGet(bldCfg.optGrpList, optGrpIdx); - String *const configOptGrp = strNew(); - - if (optGrpIdx != 0) - strCatZ(config, COMMENT_SEPARATOR "\n"); - - strCatFmt( - configOptGrp, - " PARSE_RULE_OPTION_GROUP\n" - " (\n" - " PARSE_RULE_OPTION_GROUP_NAME(\"%s\"),\n" - " ),", - strZ(optGrp->name)); - - strCat(config, bldCfgRenderLabel(configOptGrp, label, strNewFmt("opt-grp/%s", strZ(optGrp->name)))); - strCatChr(config, '\n'); - } - - strCatZ( - config, - "};\n"); - - // Option rules - // ----------------------------------------------------------------------------------------------------------------------------- - StringList *const dynamicDefaultList = strLstNew(); - - strCatZ( - config, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Option parse data\n" - COMMENT_BLOCK_END "\n"); - - strCatFmt( - config, "%s\n", - strZ( - bldDefineRender( - STRDEF("PARSE_RULE_VAL_OPT(value)"), - strNewFmt("PARSE_RULE_U32_%zu(cfgOpt##value)", bldCfgRenderVar128Size(lstSize(bldCfg.optList) - 1))))); - - strCatZ( - config, - "\n" - "static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =\n" - "{\n"); - - StringList *const cmdRoleAllList = strLstNew(); - strLstAdd(cmdRoleAllList, CMD_ROLE_MAIN_STR); - strLstAdd(cmdRoleAllList, CMD_ROLE_ASYNC_STR); - strLstAdd(cmdRoleAllList, CMD_ROLE_LOCAL_STR); - strLstAdd(cmdRoleAllList, CMD_ROLE_REMOTE_STR); - - for (unsigned int optIdx = 0; optIdx < lstSize(bldCfg.optList); optIdx++) - { - const BldCfgOption *const opt = lstGet(bldCfg.optList, optIdx); - String *const configOpt = strNew(); - - // Build list of dynamic defaults - if (opt->defaultType == defaultTypeDynamic) - strLstAddIfMissing(dynamicDefaultList, opt->defaultValue->value); - - if (optIdx != 0) - strCatZ(config, COMMENT_SEPARATOR "\n"); - - strCatFmt( - configOpt, - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"%s\"),\n" - " PARSE_RULE_OPTION_TYPE(%s),\n", - strZ(opt->name), strZ(bldEnum("", opt->type))); - - if (opt->sequence) - strCatZ(configOpt, " PARSE_RULE_OPTION_SEQUENCE(true),\n"); - - if (opt->internal) - strCatZ(configOpt, " PARSE_RULE_OPTION_INTERNAL(true)\n"); - - if (opt->defaultType == defaultTypeDynamic) - strCatZ(configOpt, " PARSE_RULE_OPTION_DEFAULT_TYPE(Dynamic),\n"); - - if (opt->boolLike) - strCatZ(configOpt, " PARSE_RULE_OPTION_BOOL_LIKE(true),\n"); - - if (opt->beta) - strCatZ(configOpt, " PARSE_RULE_OPTION_BETA(true),\n"); - - if (opt->negate) - strCatZ(configOpt, " PARSE_RULE_OPTION_NEGATE(true),\n"); - - if (opt->reset) - strCatZ(configOpt, " PARSE_RULE_OPTION_RESET(true),\n"); - - strCatFmt( - configOpt, - " PARSE_RULE_OPTION_REQUIRED(%s),\n" - " PARSE_RULE_OPTION_SECTION(%s),\n", - cvtBoolToConstZ(opt->required), strZ(bldEnum("", opt->section))); - - if (opt->secure) - strCatZ(configOpt, " PARSE_RULE_OPTION_SECURE(true),\n"); - - if (strEq(opt->type, OPT_TYPE_HASH_STR) || strEq(opt->type, OPT_TYPE_LIST_STR)) - strCatZ(configOpt, " PARSE_RULE_OPTION_MULTI(true),\n"); - - if (opt->group != NULL) - { - strCatFmt( - configOpt, - " PARSE_RULE_OPTION_GROUP_ID(%s),\n", - strZ(bldEnum("", opt->group))); - } - - // If an unindexed deprecation matches the base option name of an indexed option - if (opt->deprecateList != NULL) - { - for (unsigned int deprecateIdx = 0; deprecateIdx < lstSize(opt->deprecateList); deprecateIdx++) - { - const BldCfgOptionDeprecate *const deprecate = lstGet(opt->deprecateList, deprecateIdx); - - if (strEq(deprecate->name, opt->name) && deprecate->unindexed) - { - strCatZ(configOpt, " PARSE_RULE_OPTION_DEPRECATE_MATCH(true),\n"); - break; - } - } - } - - // Build command internal list - String *const cmdInternal = strNew(); - - for (unsigned int optCmdIdx = 0; optCmdIdx < lstSize(opt->cmdList); optCmdIdx++) - { - BldCfgOptionCommand *const optCmd = lstGet(opt->cmdList, optCmdIdx); - - if (optCmd->internal != opt->internal) - { - strCatFmt( - cmdInternal, " PARSE_RULE_OPTION_COMMAND_INTERNAL(%s, %s),\n", strZ(bldEnum("", optCmd->name)), - cvtBoolToConstZ(optCmd->internal)); - } - } - - if (strSize(cmdInternal) > 0) - { - strCatFmt( - configOpt, - "\n" - " PARSE_RULE_OPTION_COMMAND_INTERNAL_LIST\n" - " (\n" - "%s" - " )\n", - strZ(cmdInternal)); - } - - // Build command role valid lists - for (unsigned int cmdRoleAllIdx = 0; cmdRoleAllIdx < strLstSize(cmdRoleAllList); cmdRoleAllIdx++) - { - String *const configRole = strNew(); - - const String *const cmdRole = strLstGet(cmdRoleAllList, cmdRoleAllIdx); - - for (unsigned int optCmdIdx = 0; optCmdIdx < lstSize(opt->cmdList); optCmdIdx++) - { - BldCfgOptionCommand *const optCmd = lstGet(opt->cmdList, optCmdIdx); - - if (strLstExists(optCmd->roleList, cmdRole)) - { - strCatFmt( - configRole, - " PARSE_RULE_OPTION_COMMAND(%s)\n", strZ(bldEnum("", optCmd->name))); - } - } - - if (!strEmpty(configRole)) - { - strCatFmt( - configOpt, - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_%s_VALID_LIST\n" - " (\n" - "%s" - " ),\n", - strZ(strUpper(strDup(cmdRole))), strZ(configRole)); - } - } - - // Build default optional rules - KeyValue *const optionalDefaultRule = kvNew(); - const Variant *const ruleDepend = VARSTRDEF("01-depend"); - const Variant *const ruleSequence = VARSTRDEF("02-sequence"); - const Variant *const ruleAllowRange = VARSTRDEF("03-allow-range"); - const Variant *const ruleAllowList = VARSTRDEF("04-allow-list"); - const Variant *const ruleDefault = VARSTRDEF("05-default"); - const Variant *const ruleRequire = VARSTRDEF("06-require"); - const Variant *const ruleList[] = {ruleDepend, ruleSequence, ruleAllowRange, ruleAllowList, ruleDefault, ruleRequire}; - - if (opt->depend) - kvAdd(optionalDefaultRule, ruleDepend, VARSTR(bldCfgRenderValid(opt->type, opt->depend))); - - if (opt->allowRange != NULL) - { - kvAdd(optionalDefaultRule, ruleAllowRange, VARSTR(bldCfgRenderAllowRange(opt->allowRange, opt->type))); - - if (opt->allowRange->mapList != NULL) - { - for (unsigned int mapIdx = 0; mapIdx < lstSize(opt->allowRange->mapList); mapIdx++) - { - const BldCfgOptionAllowRangeMap *const allowRangeMap = lstGet(opt->allowRange->mapList, mapIdx); - - bldCfgRenderValueAdd(opt->type, false, allowRangeMap->min, ruleValMap); - bldCfgRenderValueAdd(opt->type, false, allowRangeMap->max, ruleValMap); - } - } - else - { - bldCfgRenderValueAdd(opt->type, false, opt->allowRange->min, ruleValMap); - bldCfgRenderValueAdd(opt->type, false, opt->allowRange->max, ruleValMap); - } - } - - if (opt->allowList != NULL) - { - kvAdd(optionalDefaultRule, ruleAllowList, VARSTR(bldCfgRenderAllowList(opt->allowList, opt->type))); - - for (unsigned int allowIdx = 0; allowIdx < lstSize(opt->allowList); allowIdx++) - { - bldCfgRenderValueAdd( - opt->type, false, ((const BldCfgOptionValue *)lstGet(opt->allowList, allowIdx))->value, ruleValMap); - } - } - - if (opt->defaultValue != NULL) - { - kvAdd( - optionalDefaultRule, ruleDefault, - VARSTR(bldCfgRenderDefault(opt->defaultValue, opt->defaultType, opt->type, opt->sequence, opt->allowList))); - - if (!strEq(opt->type, OPT_TYPE_BOOLEAN_STR)) - { - if (opt->defaultValue->value != NULL) - bldCfgRenderValueAdd(opt->type, opt->defaultType, opt->defaultValue->value, ruleValMap); - - if (opt->defaultValue->mapList != NULL) - { - for (unsigned int mapIdx = 0; mapIdx < lstSize(opt->defaultValue->mapList); mapIdx++) - { - const BldCfgOptionDefaultMap *const map = lstGet(opt->defaultValue->mapList, mapIdx); - bldCfgRenderValueAdd(opt->type, opt->defaultType, map->value, ruleValMap); - } - } - } - } - - // Build command optional rules - KeyValue *const optionalCmdRule = kvNew(); - - for (unsigned int optCmdIdx = 0; optCmdIdx < lstSize(opt->cmdList); optCmdIdx++) - { - BldCfgOptionCommand *const optCmd = lstGet(opt->cmdList, optCmdIdx); - KeyValue *const optionalCmdRuleType = kvNew(); - - // Depends - if (optCmd->depend != NULL) - kvAdd(optionalCmdRuleType, ruleDepend, VARSTR(bldCfgRenderValid(opt->type, optCmd->depend))); - - // Sequences - if (optCmd->sequence != opt->sequence) - { - kvAdd( - optionalCmdRuleType, ruleSequence, - VARSTR(strNewFmt(" PARSE_RULE_OPTIONAL_%sSEQUENCE()", optCmd->required ? "" : "NOT_"))); - } - - // Allow lists - if (optCmd->allowList != NULL) - { - kvAdd(optionalCmdRuleType, ruleAllowList, VARSTR(bldCfgRenderAllowList(optCmd->allowList, opt->type))); - - for (unsigned int allowIdx = 0; allowIdx < lstSize(optCmd->allowList); allowIdx++) - { - bldCfgRenderValueAdd( - opt->type, false, ((const BldCfgOptionValue *)lstGet(optCmd->allowList, allowIdx))->value, ruleValMap); - } - } - - // Defaults - if (optCmd->defaultValue != NULL) - { - kvAdd( - optionalCmdRuleType, ruleDefault, - VARSTR( - bldCfgRenderDefault(optCmd->defaultValue, opt->defaultType, opt->type, opt->sequence, optCmd->allowList))); - - if (!strEq(opt->type, OPT_TYPE_BOOLEAN_STR)) - bldCfgRenderValueAdd(opt->type, opt->defaultType, optCmd->defaultValue->value, ruleValMap); - } - - // Requires - if (optCmd->required != opt->required) - { - kvAdd( - optionalCmdRuleType, ruleRequire, - VARSTR(strNewFmt(" PARSE_RULE_OPTIONAL_%sREQUIRED()", optCmd->required ? "" : "NOT_"))); - } - - // Add defaults - if (varLstSize(kvKeyList(optionalCmdRuleType)) > 0) - { - for (unsigned int ruleIdx = 0; ruleIdx < LENGTH_OF(ruleList); ruleIdx++) - { - if (kvKeyExists(optionalCmdRuleType, ruleList[ruleIdx])) - kvAdd(optionalCmdRule, VARSTR(optCmd->name), kvGet(optionalCmdRuleType, ruleList[ruleIdx])); - else if (kvKeyExists(optionalDefaultRule, ruleList[ruleIdx])) - kvAdd(optionalCmdRule, VARSTR(optCmd->name), kvGet(optionalDefaultRule, ruleList[ruleIdx])); - } - } - } - - // Add optional rules - unsigned int optionalCmdRuleSize = varLstSize(kvKeyList(optionalCmdRule)); - unsigned int optionalDefaultRuleSize = varLstSize(kvKeyList(optionalDefaultRule)); - - if (optionalCmdRuleSize != 0 || optionalDefaultRuleSize != 0) - { - strCatZ( - configOpt, - "\n" - " PARSE_RULE_OPTIONAL\n" - " (\n"); - - if (optionalCmdRuleSize != 0) - { - KeyValue *const combine = kvNew(); - - for (unsigned int ruleIdx = 0; ruleIdx < optionalCmdRuleSize; ruleIdx++) - { - const Variant *const cmd = varLstGet(kvKeyList(optionalCmdRule), ruleIdx); - const VariantList *const groupList = kvGetList(optionalCmdRule, cmd); - String *const group = strNew(); - - for (unsigned int groupIdx = 0; groupIdx < varLstSize(groupList); groupIdx++) - { - strCatFmt( - group, - "\n" - "%s,\n", - strZ(varStr(varLstGet(groupList, groupIdx)))); - } - - kvAdd(combine, VARSTR(group), cmd); - } - - unsigned int combineSize = varLstSize(kvKeyList(combine)); - - for (unsigned int ruleIdx = 0; ruleIdx < combineSize; ruleIdx++) - { - const Variant *const group = varLstGet(kvKeyList(combine), ruleIdx); - const VariantList *const cmdList = kvGetList(combine, group); - - if (ruleIdx != 0) - strCatChr(configOpt, '\n'); - - strCatZ( - configOpt, - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_FILTER_CMD\n" - " (\n"); - - for (unsigned int cmdIdx = 0; cmdIdx < varLstSize(cmdList); cmdIdx++) - { - strCatFmt( - configOpt, - " PARSE_RULE_VAL_CMD(%s),\n", - strZ(bldEnum("", varStr(varLstGet(cmdList, cmdIdx))))); - } - - strCatFmt( - configOpt, - " ),\n" - "%s" - " ),\n", - strZ(varStr(group))); - } - } - - if (optionalDefaultRuleSize != 0) - { - if (optionalCmdRuleSize != 0) - strCatChr(configOpt, '\n'); - - strCatZ( - configOpt, - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n"); - - for (unsigned int ruleIdx = 0; ruleIdx < optionalDefaultRuleSize; ruleIdx++) - { - const Variant *const key = varLstGet(kvKeyList(optionalDefaultRule), ruleIdx); - - if (ruleIdx != 0) - strCatChr(configOpt, '\n'); - - strCatFmt( - configOpt, - "%s,\n", - strZ(varStr(kvGet(optionalDefaultRule, key)))); - } - - strCatZ(configOpt, " ),\n"); - } - - strCatZ(configOpt, " ),\n"); - } - - strCatZ(configOpt, " ),"); - - // Add option to config - strCat(config, bldCfgRenderLabel(configOpt, label, strNewFmt("opt/%s", strZ(opt->name)))); - strCatChr(config, '\n'); - } - - strCatZ( - config, - "};\n"); - - // Option deprecations - // ----------------------------------------------------------------------------------------------------------------------------- - List *const deprecateCombineList = lstNewP(sizeof(BldCfgRenderOptionDeprecate), .comparator = lstComparatorStr); - - for (unsigned int optIdx = 0; optIdx < lstSize(bldCfg.optList); optIdx++) - { - const BldCfgOption *const opt = lstGet(bldCfg.optList, optIdx); - - if (opt->deprecateList != NULL) - { - for (unsigned int deprecateIdx = 0; deprecateIdx < lstSize(opt->deprecateList); deprecateIdx++) - { - const BldCfgOptionDeprecate *const deprecate = lstGet(opt->deprecateList, deprecateIdx); - const BldCfgRenderOptionDeprecate bldCfgRenderOptionDeprecate = - { - .name = deprecate->name, - .option = opt, - .indexed = deprecate->indexed, - .unindexed = deprecate->unindexed, - }; - - lstAdd(deprecateCombineList, &bldCfgRenderOptionDeprecate); - } - } - } - - lstSort(deprecateCombineList, sortOrderAsc); - - strCatFmt( - config, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Option deprecations\n" - COMMENT_BLOCK_END "\n" - "%s\n" - "\n" - "static const ParseRuleOptionDeprecate parseRuleOptionDeprecate[CFG_OPTION_DEPRECATE_TOTAL] =\n" - "{\n", - strZ(bldDefineRender(STRDEF("CFG_OPTION_DEPRECATE_TOTAL"), strNewFmt("%u", lstSize(deprecateCombineList))))); - - for (unsigned int deprecateIdx = 0; deprecateIdx < lstSize(deprecateCombineList); deprecateIdx++) - { - const BldCfgRenderOptionDeprecate *const deprecate = lstGet(deprecateCombineList, deprecateIdx); - String *const configOptDep = strNew(); - - if (deprecateIdx != 0) - strCatZ(config, COMMENT_SEPARATOR "\n"); - - strCatFmt( - configOptDep, - " {\n" - " .name = \"%s\",\n" - " .id = %s,\n", - strZ(deprecate->name), strZ(bldEnum("cfgOpt", deprecate->option->name))); - - if (deprecate->indexed) - strCatZ(configOptDep, " .indexed = true,\n"); - - if (deprecate->unindexed) - strCatZ(configOptDep, " .unindexed = true,\n"); - - strCatZ(configOptDep, " },"); - - strCat(config, bldCfgRenderLabel(configOptDep, label, strNewFmt("opt-deprecate/%s", strZ(deprecate->option->name)))); - strCatChr(config, '\n'); - } - - strCatZ(config, "};\n"); - - // Order for option parse resolution - // ----------------------------------------------------------------------------------------------------------------------------- - strCatZ( - config, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Order for option parse resolution\n" - COMMENT_BLOCK_END "\n" - "static const uint8_t optionResolveOrder[] =\n" - "{\n"); - - // Render resolve order - String *const configResolve = strNew(); - - for (unsigned int optResolveIdx = 0; optResolveIdx < lstSize(bldCfg.optResolveList); optResolveIdx++) - { - bldCfgRenderLf(configResolve, optResolveIdx != 0); - - strCatFmt( - configResolve, " %s,", - strZ(bldEnum("cfgOpt", (*(BldCfgOption **)lstGet(bldCfg.optResolveList, optResolveIdx))->name))); - } - - strCat(config, bldCfgRenderLabel(configResolve, label, STRDEF("opt-resolve-order"))); - strCatZ(config, "\n};\n"); - - // Rule values - // ----------------------------------------------------------------------------------------------------------------------------- - String *const configVal = strNew(); - - strCat( - configVal, - bldCfgRenderValueRender(OPT_TYPE_STRING_STR, ruleValMap, label, "StringPubConst", "String", "Str", "STR", "val/str")); - strCat(configVal, bldCfgRenderValueRender(OPT_TYPE_INTEGER_STR, ruleValMap, label, "int", "Int", "Int", "INT", "val/int")); - strCat(configVal, bldCfgRenderValueRender(OPT_TYPE_SIZE_STR, ruleValMap, label, "int64_t", "Size", "Size", "SIZE", "val/size")); - strCat( - configVal, - bldCfgRenderValueRender(OPT_TYPE_TIME_STR, ruleValMap, label, "unsigned int", "Time", "Time", "TIME", "val/time")); - - // Dynamic default values - // ----------------------------------------------------------------------------------------------------------------------------- - strCatZ( - configVal, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Dynamic default values\n" - COMMENT_BLOCK_END "\n"); - - strCatFmt( - configVal, "%s\n", - strZ( - bldDefineRender( - STRDEF("PARSE_RULE_DEFAULT_DYNAMIC(value)"), - strNewFmt( - "PARSE_RULE_U32_%zu(parseRuleDefaultDynamic##value)", - bldCfgRenderVar128Size(strLstSize(dynamicDefaultList) - 1))))); - - strCatZ( - configVal, - "\ntypedef enum\n" - "{\n"); - - for (unsigned int dynamicDefaultIdx = 0; dynamicDefaultIdx < strLstSize(dynamicDefaultList); dynamicDefaultIdx++) - { - strCatFmt( - configVal, " %s,\n", - zNewFmt("parseRuleDefaultDynamic%s", strZ(bldEnum("", strLstGet(dynamicDefaultList, dynamicDefaultIdx))))); - } - - strCatZ( - configVal, - "} ParseRuleDefaultDynamic;\n"); - - // Write to storage - // ----------------------------------------------------------------------------------------------------------------------------- - bldPut( - storageRepo, "src/config/parse.auto.c.inc", - BUFSTR(strNewFmt("%s%s%s", strZ(bldHeader(CONFIG_MODULE, PARSE_AUTO_COMMENT)), strZ(configVal), strZ(config)))); -} - -/**********************************************************************************************************************************/ -void -bldCfgRender(const Storage *const storageRepo, const BldCfg bldCfg, const bool label) -{ - bldCfgRenderConfigAutoH(storageRepo, bldCfg); - bldCfgRenderParseAutoC(storageRepo, bldCfg, label); -} diff --git a/src/build/config/render.h b/src/build/config/render.h deleted file mode 100644 index 598ff80f1..000000000 --- a/src/build/config/render.h +++ /dev/null @@ -1,15 +0,0 @@ -/*********************************************************************************************************************************** -Render Configuration Data -***********************************************************************************************************************************/ -#ifndef BUILD_CONFIG_RENDER_H -#define BUILD_CONFIG_RENDER_H - -#include "build/config/parse.h" - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Render auto-generated config files -void bldCfgRender(const Storage *const storageRepo, const BldCfg bldCfg, bool label); - -#endif diff --git a/src/build/error/parse.c b/src/build/error/parse.c deleted file mode 100644 index ef5f6c6f2..000000000 --- a/src/build/error/parse.c +++ /dev/null @@ -1,105 +0,0 @@ -/*********************************************************************************************************************************** -Parse Error Yaml -***********************************************************************************************************************************/ -#include - -#include - -#include "common/log.h" -#include "common/macro.h" -#include "common/type/convert.h" -#include "storage/posix/storage.h" - -#include "build/common/yaml.h" -#include "build/error/parse.h" - -/*********************************************************************************************************************************** -Error min/max codes -***********************************************************************************************************************************/ -#define ERROR_CODE_MIN 25 -#define ERROR_CODE_MAX 125 - -/*********************************************************************************************************************************** -Parse error list -***********************************************************************************************************************************/ -typedef struct BldErrErrorRaw -{ - const String *const name; // See BldErrError for comments - unsigned int code; - bool fatal; -} BldErrErrorRaw; - -static List * -bldErrParseErrorList(Yaml *const yaml) -{ - List *const result = lstNewP(sizeof(BldErrError), .comparator = lstComparatorStr); - - MEM_CONTEXT_TEMP_BEGIN() - { - YAML_MAP_BEGIN(yaml) - { - BldErrErrorRaw errRaw = {.name = yamlScalarNext(yaml).value}; - - // If scalar then it is an error code - if (yamlEventPeekIs(yaml, yamlEventTypeScalar)) - { - errRaw.code = cvtZToUInt(strZ(yamlScalarNext(yaml).value)); - } - // Else there may be multiple definitions - else - { - YAML_MAP_BEGIN(yaml) - { - const String *const errDef = yamlScalarNext(yaml).value; - const YamlEvent errDefVal = yamlScalarNext(yaml); - - if (strEqZ(errDef, "code")) - errRaw.code = cvtZToUInt(strZ(errDefVal.value)); - else if (strEqZ(errDef, "fatal")) - errRaw.fatal = yamlBoolParse(errDefVal); - else - THROW_FMT(FormatError, "unknown error definition '%s'", strZ(errDef)); - } - YAML_MAP_END(); - } - - if (errRaw.code < ERROR_CODE_MIN || errRaw.code > ERROR_CODE_MAX) - { - THROW_FMT( - FormatError, "error '%s' code must be >= " STRINGIFY(ERROR_CODE_MIN) " and <= " STRINGIFY(ERROR_CODE_MAX), - strZ(errRaw.name)); - } - - // Add to list - MEM_CONTEXT_BEGIN(lstMemContext(result)) - { - const BldErrError bldErrError = - { - .name = strDup(errRaw.name), - .code = errRaw.code, - .fatal = errRaw.fatal, - }; - - lstAdd(result, &bldErrError); - } - MEM_CONTEXT_END(); - } - YAML_MAP_END(); - } - MEM_CONTEXT_TEMP_END(); - - return result; -} - -/**********************************************************************************************************************************/ -BldErr -bldErrParse(const Storage *const storageRepo) -{ - // Initialize yaml - Yaml *const yaml = yamlNew(storageGetP(storageNewReadP(storageRepo, STRDEF("src/build/error/error.yaml")))); - - // Parse error - const List *const errList = bldErrParseErrorList(yaml); - - return (BldErr){.errList = errList}; -} diff --git a/src/build/error/parse.h b/src/build/error/parse.h deleted file mode 100644 index 4125ade77..000000000 --- a/src/build/error/parse.h +++ /dev/null @@ -1,30 +0,0 @@ -/*********************************************************************************************************************************** -Parse Error Yaml -***********************************************************************************************************************************/ -#ifndef BUILD_ERROR_PARSE_H -#define BUILD_ERROR_PARSE_H - -#include "common/type/string.h" - -/*********************************************************************************************************************************** -Types -***********************************************************************************************************************************/ -typedef struct BldErrError -{ - const String *name; // Name - unsigned int code; // Code - bool fatal; // Is the error fatal? -} BldErrError; - -typedef struct BldErr -{ - const List *errList; // Command list -} BldErr; - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Parse error.yaml -BldErr bldErrParse(const Storage *const storageRepo); - -#endif diff --git a/src/build/error/render.c b/src/build/error/render.c deleted file mode 100644 index aefb4b361..000000000 --- a/src/build/error/render.c +++ /dev/null @@ -1,142 +0,0 @@ -/*********************************************************************************************************************************** -Render Error Data -***********************************************************************************************************************************/ -#include - -#include - -#include "common/log.h" -#include "common/type/convert.h" -#include "storage/posix/storage.h" - -#include "build/common/render.h" -#include "build/error/render.h" - -/*********************************************************************************************************************************** -Build error name from a string -***********************************************************************************************************************************/ -static String * -bldErrName(const String *const value) -{ - String *const result = strNew(); - const char *const valuePtr = strZ(value); - - bool upper = true; - - for (unsigned int valueIdx = 0; valueIdx < strSize(value); valueIdx++) - { - strCatChr(result, upper ? (char)toupper(valuePtr[valueIdx]) : valuePtr[valueIdx]); - upper = false; - - if (valuePtr[valueIdx + 1] == '-') - { - upper = true; - valueIdx++; - } - } - - strCatZ(result, "Error"); - - return result; -} - -/*********************************************************************************************************************************** -Render error.auto.h -***********************************************************************************************************************************/ -#define ERROR_MODULE "error" -#define ERROR_AUTO_COMMENT "Error Type Definition" - -static void -bldErrRenderErrorAutoH(const Storage *const storageRepo, const BldErr bldErr) -{ - String *error = strCatFmt( - strNew(), - "%s" - "#ifndef COMMON_ERROR_ERROR_AUTO_H\n" - "#define COMMON_ERROR_ERROR_AUTO_H\n", - strZ(bldHeader(ERROR_MODULE, ERROR_AUTO_COMMENT))); - - // Error declarations - // ----------------------------------------------------------------------------------------------------------------------------- - strCatZ( - error, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Error type declarations\n" - COMMENT_BLOCK_END "\n"); - - for (unsigned int errIdx = 0; errIdx < lstSize(bldErr.errList); errIdx++) - { - const BldErrError *const err = lstGet(bldErr.errList, errIdx); - - strCatFmt(error, "ERROR_DECLARE(%s);\n", strZ(bldErrName(err->name))); - } - - // End and save - strCatZ( - error, - "\n" - "#endif\n"); - - bldPut(storageRepo, "src/common/error/error.auto.h", BUFSTR(error)); -} - -/*********************************************************************************************************************************** -Render error.auto.c.inc -***********************************************************************************************************************************/ -static void -bldErrRenderErrorAutoC(const Storage *const storageRepo, const BldErr bldErr) -{ - String *error = bldHeader(ERROR_MODULE, ERROR_AUTO_COMMENT); - - // Error type definitions - // ----------------------------------------------------------------------------------------------------------------------------- - strCatZ( - error, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Error type definitions\n" - COMMENT_BLOCK_END "\n"); - - for (unsigned int errIdx = 0; errIdx < lstSize(bldErr.errList); errIdx++) - { - const BldErrError *const err = lstGet(bldErr.errList, errIdx); - - strCatFmt( - error, "ERROR_DEFINE(%3u, %s, %s, RuntimeError);\n", err->code, strZ(bldErrName(err->name)), - cvtBoolToConstZ(err->fatal)); - } - - // Error type array - // ----------------------------------------------------------------------------------------------------------------------------- - strCatZ( - error, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Error type array\n" - COMMENT_BLOCK_END "\n" - "static const ErrorType *const errorTypeList[] =\n" - "{\n"); - - for (unsigned int errIdx = 0; errIdx < lstSize(bldErr.errList); errIdx++) - { - const BldErrError *const err = lstGet(bldErr.errList, errIdx); - - strCatFmt(error, " &%s,\n", strZ(bldErrName(err->name))); - } - - strCatZ( - error, - " NULL,\n" - "};\n"); - - bldPut(storageRepo, "src/common/error/error.auto.c.inc", BUFSTR(error)); -} - -/**********************************************************************************************************************************/ -void -bldErrRender(const Storage *const storageRepo, const BldErr bldErr) -{ - bldErrRenderErrorAutoH(storageRepo, bldErr); - bldErrRenderErrorAutoC(storageRepo, bldErr); -} diff --git a/src/build/error/render.h b/src/build/error/render.h deleted file mode 100644 index 0abde055e..000000000 --- a/src/build/error/render.h +++ /dev/null @@ -1,15 +0,0 @@ -/*********************************************************************************************************************************** -Render Error Data -***********************************************************************************************************************************/ -#ifndef BUILD_ERROR_RENDER_H -#define BUILD_ERROR_RENDER_H - -#include "build/error/parse.h" - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Render auto-generated error files -void bldErrRender(const Storage *const storageRepo, const BldErr bldErr); - -#endif diff --git a/src/build/help/parse.c b/src/build/help/parse.c deleted file mode 100644 index d13b6b4cb..000000000 --- a/src/build/help/parse.c +++ /dev/null @@ -1,244 +0,0 @@ -/*********************************************************************************************************************************** -Parse Help Xml -***********************************************************************************************************************************/ -#include - -#include "storage/storage.h" - -#include "build/config/parse.h" -#include "build/help/parse.h" - -/*********************************************************************************************************************************** -Parse option help -***********************************************************************************************************************************/ -// Helper to parse options -static void -bldHlpParseOption(XmlNodeList *const xmlOptList, List *const optList, const String *const sectionDefault) -{ - ASSERT(xmlOptList != NULL); - ASSERT(optList != NULL); - - for (unsigned int optIdx = 0; optIdx < xmlNodeLstSize(xmlOptList); optIdx++) - { - const XmlNode *const xmlOpt = xmlNodeLstGet(xmlOptList, optIdx); - - // Get section or use default - const String *section = xmlNodeAttribute(xmlOpt, STRDEF("section")); - - if (section == NULL) - section = sectionDefault; - - // Add option to list - MEM_CONTEXT_BEGIN(lstMemContext(optList)) - { - BldHlpOption bldHlpOption = - { - .name = xmlNodeAttribute(xmlOpt, STRDEF("id")), - .section = strDup(section), - .title = xmlNodeAttribute(xmlOpt, STRDEF("name")), - .summary = xmlNodeChild(xmlOpt, STRDEF("summary"), true), - .description = xmlNodeChild(xmlOpt, STRDEF("text"), true), - }; - - // Add examples - const XmlNodeList *const exampleList = xmlNodeChildList(xmlOpt, STRDEF("example")); - - if (xmlNodeLstSize(exampleList) > 0) - { - StringList *const exampleListBuild = strLstNew(); - - for (unsigned int exampleIdx = 0; exampleIdx < xmlNodeLstSize(exampleList); exampleIdx++) - strLstAdd(exampleListBuild, xmlNodeContent(xmlNodeLstGet(exampleList, exampleIdx))); - - bldHlpOption.exampleList = exampleListBuild; - } - - lstAdd(optList, &bldHlpOption); - } - MEM_CONTEXT_END(); - } - - lstSort(optList, sortOrderAsc); -} - -static List * -bldHlpParseOptionList(XmlNode *const xml) -{ - List *const result = lstNewP(sizeof(BldHlpOption), .comparator = lstComparatorStr); - - // Parse config options - const XmlNodeList *xmlSectionList = xmlNodeChildList( - xmlNodeChild(xmlNodeChild(xml, STRDEF("config"), true), STRDEF("config-section-list"), true), STRDEF("config-section")); - - for (unsigned int sectionIdx = 0; sectionIdx < xmlNodeLstSize(xmlSectionList); sectionIdx++) - { - const XmlNode *const xmlSection = xmlNodeLstGet(xmlSectionList, sectionIdx); - - bldHlpParseOption( - xmlNodeChildList(xmlNodeChild(xmlSection, STRDEF("config-key-list"), true), STRDEF("config-key")), result, - xmlNodeAttribute(xmlSection, STRDEF("id"))); - } - - // Parse command-line only options - bldHlpParseOption( - xmlNodeChildList( - xmlNodeChild( - xmlNodeChild(xmlNodeChild(xml, STRDEF("operation"), true), STRDEF("operation-general"), true), - STRDEF("option-list"), true), - STRDEF("option")), - result, NULL); - - return result; -} - -/*********************************************************************************************************************************** -Parse command help -***********************************************************************************************************************************/ -static List * -bldHlpParseCommandList(XmlNode *const xml) -{ - List *const result = lstNewP(sizeof(BldHlpCommand), .comparator = lstComparatorStr); - - // Parse commands - const XmlNodeList *xmlCmdList = xmlNodeChildList(xml, STRDEF("command")); - - for (unsigned int cmdIdx = 0; cmdIdx < xmlNodeLstSize(xmlCmdList); cmdIdx++) - { - const XmlNode *const xmlCmd = xmlNodeLstGet(xmlCmdList, cmdIdx); - - // Parse option list if any - List *cmdOptList = NULL; - const XmlNode *const xmlCmdOptListParent = xmlNodeChild(xmlCmd, STRDEF("option-list"), false); - - if (xmlCmdOptListParent != NULL) - { - cmdOptList = lstNewP(sizeof(BldHlpOption), .comparator = lstComparatorStr); - - bldHlpParseOption(xmlNodeChildList(xmlCmdOptListParent, STRDEF("option")), cmdOptList, NULL); - } - - // Add command to list - MEM_CONTEXT_BEGIN(lstMemContext(result)) - { - const BldHlpCommand bldHlpCommand = - { - .name = xmlNodeAttribute(xmlCmd, STRDEF("id")), - .title = xmlNodeAttribute(xmlCmd, STRDEF("name")), - .summary = xmlNodeChild(xmlCmd, STRDEF("summary"), true), - .description = xmlNodeChild(xmlCmd, STRDEF("text"), true), - .optList = lstMove(cmdOptList, memContextCurrent()), - }; - - lstAdd(result, &bldHlpCommand); - } - MEM_CONTEXT_END(); - } - - lstSort(result, sortOrderAsc); - - return result; -} - -/*********************************************************************************************************************************** -Build section list -***********************************************************************************************************************************/ -static List * -bldHlpParseSectionList(XmlNode *const xml, const bool detail) -{ - List *const result = lstNewP(sizeof(BldHlpSection), .comparator = lstComparatorStr); - - // Build section list - const XmlNodeList *const xmlSectionList = xmlNodeChildList( - xmlNodeChild(xmlNodeChild(xml, STRDEF("config"), true), STRDEF("config-section-list"), true), STRDEF("config-section")); - - for (unsigned int sectionIdx = 0; sectionIdx < xmlNodeLstSize(xmlSectionList); sectionIdx++) - { - const XmlNode *const xmlSection = xmlNodeLstGet(xmlSectionList, sectionIdx); - const BldHlpSection bldHlpSection = - { - .id = xmlNodeAttribute(xmlSection, STRDEF("id")), - .name = xmlNodeAttribute(xmlSection, STRDEF("name")), - .introduction = xmlNodeChild(xmlSection, STRDEF("text"), detail), - }; - - lstAdd(result, &bldHlpSection); - } - - lstSort(result, sortOrderAsc); - - return result; -} - -/*********************************************************************************************************************************** -Reconcile help -***********************************************************************************************************************************/ -static void -bldHlpValidate(const BldHlp bldHlp, const BldCfg bldCfg) -{ - // Validate command help - for (unsigned int cmdIdx = 0; cmdIdx < lstSize(bldCfg.cmdList); cmdIdx++) - { - const BldCfgCommand *const cmd = lstGet(bldCfg.cmdList, cmdIdx); - const BldHlpCommand *const cmdHlp = lstFind(bldHlp.cmdList, &cmd->name); - - if (cmdHlp == NULL) - THROW_FMT(FormatError, "command '%s' must have help", strZ(cmd->name)); - } - - // Validate option help - for (unsigned int optIdx = 0; optIdx < lstSize(bldCfg.optList); optIdx++) - { - const BldCfgOption *const opt = lstGet(bldCfg.optList, optIdx); - const BldHlpOption *const optHlp = lstFind(bldHlp.optList, &opt->name); - - // If help was not found in general command-line or config options then check command overrides - if (optHlp == NULL) - { - for (unsigned int optCmdListIdx = 0; optCmdListIdx < lstSize(opt->cmdList); optCmdListIdx++) - { - const BldCfgOptionCommand *const optCmd = lstGet(opt->cmdList, optCmdListIdx); - const BldHlpCommand *const cmdHlp = lstFind(bldHlp.cmdList, &optCmd->name); - CHECK(AssertError, cmdHlp != NULL, "command help is NULL"); - - // Only options with a command role of main require help - if (!strLstExists(optCmd->roleList, CMD_ROLE_MAIN_STR)) - continue; - - const BldHlpOption *const cmdOptHlp = cmdHlp->optList != NULL ? lstFind(cmdHlp->optList, &opt->name) : NULL; - - if (cmdOptHlp == NULL) - THROW_FMT(FormatError, "option '%s' must have help for command '%s'", strZ(opt->name), strZ(optCmd->name)); - } - } - } -} - -/**********************************************************************************************************************************/ -BldHlp -bldHlpParse(const Storage *const storageRepo, const BldCfg bldCfg, const bool detail) -{ - // Initialize xml - XmlNode *const xml = xmlDocumentRoot( - xmlDocumentNewBuf(storageGetP(storageNewReadP(storageRepo, STRDEF("src/build/help/help.xml"))))); - - // Parse help - BldHlp result = - { - .sctList = bldHlpParseSectionList(xml, detail), - - .cmdTitle = xmlNodeAttribute(xmlNodeChild(xml, STRDEF("operation"), true), STRDEF("title")), - .cmdDescription = xmlNodeContent(xmlNodeChild(xmlNodeChild(xml, STRDEF("operation"), true), STRDEF("description"), detail)), - .cmdIntroduction = xmlNodeChild(xmlNodeChild(xml, STRDEF("operation"), true), STRDEF("text"), detail), - .cmdList = bldHlpParseCommandList( - xmlNodeChild(xmlNodeChild(xml, STRDEF("operation"), true), STRDEF("command-list"), true)), - - .optTitle = xmlNodeAttribute(xmlNodeChild(xml, STRDEF("config"), true), STRDEF("title")), - .optDescription = xmlNodeContent(xmlNodeChild(xmlNodeChild(xml, STRDEF("config"), true), STRDEF("description"), detail)), - .optIntroduction = xmlNodeChild(xmlNodeChild(xml, STRDEF("config"), true), STRDEF("text"), detail), - .optList = bldHlpParseOptionList(xml) - }; - - bldHlpValidate(result, bldCfg); - - return result; -} diff --git a/src/build/help/parse.h b/src/build/help/parse.h deleted file mode 100644 index d81b93301..000000000 --- a/src/build/help/parse.h +++ /dev/null @@ -1,60 +0,0 @@ -/*********************************************************************************************************************************** -Parse Help Xml -***********************************************************************************************************************************/ -#ifndef BUILD_HELP_PARSE_H -#define BUILD_HELP_PARSE_H - -#include "build/common/xml.h" -#include "build/config/parse.h" - -/*********************************************************************************************************************************** -Types -***********************************************************************************************************************************/ -typedef struct BldHlpSection -{ - const String *id; // Id - const String *name; // Name - const XmlNode *introduction; // Introduction -} BldHlpSection; - -typedef struct BldHlpCommand -{ - const String *name; // Name - const String *title; // Title - const XmlNode *summary; // Summary - const XmlNode *description; // Description - const List *optList; // Option list -} BldHlpCommand; - -typedef struct BldHlpOption -{ - const String *name; // Name - const String *section; // Section - const String *title; // Title - const XmlNode *summary; // Summary - const XmlNode *description; // Description - const StringList *exampleList; // Examples -} BldHlpOption; - -typedef struct BldHlp -{ - const List *sctList; // Section list - - const String *cmdTitle; // Command title - const String *cmdDescription; // Command description - const XmlNode *cmdIntroduction; // Command introduction - const List *cmdList; // Command list - - const String *optTitle; // Option title - const String *optDescription; // Option description - const XmlNode *optIntroduction; // Option introduction - const List *optList; // Option list -} BldHlp; - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Parse help.xml -BldHlp bldHlpParse(const Storage *const storageRepo, const BldCfg bldCfg, bool detail); - -#endif diff --git a/src/build/help/render.c b/src/build/help/render.c deleted file mode 100644 index abf55bedb..000000000 --- a/src/build/help/render.c +++ /dev/null @@ -1,370 +0,0 @@ -/*********************************************************************************************************************************** -Render Help -***********************************************************************************************************************************/ -#include - -#include - -#include -#include - -#include "common/compress/bz2/compress.h" -#include "common/io/bufferRead.h" -#include "common/io/bufferWrite.h" -#include "common/log.h" -#include "common/type/pack.h" - -#include "build/common/render.h" -#include "build/config/parse.h" -#include "build/help/parse.h" -#include "build/help/render.h" - -/*********************************************************************************************************************************** -Render xml as text -***********************************************************************************************************************************/ -static String * -bldHlpRenderReplace(const String *const string, const char *const replace, const char *const with) -{ - String *result = strNew(); - - StringList *const stringList = strLstNewSplitZ(string, replace); - - for (unsigned int stringIdx = 0; stringIdx < strLstSize(stringList); stringIdx++) - { - if (stringIdx != 0) - strCatZ(result, with); - - strCat(result, strLstGet(stringList, stringIdx)); - } - - return result; -} - -static String * -bldHlpRenderXmlNode(const xmlNodePtr xml) -{ - String *const result = strNew(); - - for (xmlNodePtr currentNode = xml->children; currentNode != NULL; currentNode = currentNode->next) - { - const String *const name = STR((const char *)currentNode->name); - - if (currentNode->type == XML_ELEMENT_NODE) - { - if (strEq(name, STRDEF("admonition"))) - { - strCatZ(result, "NOTE: "); - strCat(result, bldHlpRenderXmlNode(currentNode)); - strCatZ(result, "\n\n"); - } - else if (strEq(name, STRDEF("backrest"))) - strCatZ(result, "pgBackRest"); - else if (strEq(name, STRDEF("list"))) - { - strCat(result, bldHlpRenderXmlNode(currentNode)); - strCatChr(result, '\n'); - } - else if (strEq(name, STRDEF("list-item"))) - { - strCatZ(result, "* "); - strCat(result, bldHlpRenderXmlNode(currentNode)); - strCatChr(result, '\n'); - } - else if (strEq(name, STRDEF("p"))) - { - strCat(result, bldHlpRenderXmlNode(currentNode)); - strCatZ(result, "\n\n"); - } - else if (strEq(name, STRDEF("postgres"))) - strCatZ(result, "PostgreSQL"); - else if (strEq(name, STRDEF("quote"))) - { - strCatChr(result, '"'); - strCat(result, bldHlpRenderXmlNode(currentNode)); - strCatChr(result, '"'); - } - else if ( - strEq(name, STRDEF("id")) || strEq(name, STRDEF("br-option")) || strEq(name, STRDEF("cmd")) || - strEq(name, STRDEF("link")) || strEq(name, STRDEF("setting")) || strEq(name, STRDEF("pg-setting")) || - strEq(name, STRDEF("code")) || strEq(name, STRDEF("i")) || strEq(name, STRDEF("file")) || - strEq(name, STRDEF("path")) || strEq(name, STRDEF("b")) || strEq(name, STRDEF("host")) || - strEq(name, STRDEF("exe")) || strEq(name, STRDEF("proper"))) - { - strCat(result, bldHlpRenderXmlNode(currentNode)); - } - else - THROW_FMT(FormatError, "unknown tag '%s'", strZ(name)); - } - else if (currentNode->type == XML_TEXT_NODE) - { - xmlChar *content = xmlNodeGetContent(currentNode); - String *text = strNewZ((char *)content); - xmlFree(content); - - if (strchr(strZ(text), '\n') != NULL) - { - if (!strEmpty(strTrim(strDup(text)))) - THROW_FMT(FormatError, "text '%s' is invalid", strZ(text)); - - continue; - } - - text = bldHlpRenderReplace(text, "{[dash]}", "-"); - - strCat(result, text); - } - } - - return result; -} - -String * -bldHlpRenderXml(const XmlNode *const xml) -{ - return strTrim(bldHlpRenderXmlNode(*(const xmlNodePtr *)xml)); -} - -/*********************************************************************************************************************************** -Render help to a pack -***********************************************************************************************************************************/ -static PackWrite * -bldHlpRenderHelpAutoCPack(const BldCfg bldCfg, const BldHlp bldHlp) -{ - PackWrite *const pack = pckWriteNewP(.size = 65 * 1024); - - // Command help - // ----------------------------------------------------------------------------------------------------------------------------- - pckWriteArrayBeginP(pack); - - for (unsigned int cmdIdx = 0; cmdIdx < lstSize(bldCfg.cmdList); cmdIdx++) - { - const BldCfgCommand *const cmd = lstGet(bldCfg.cmdList, cmdIdx); - const BldHlpCommand *const cmdHlp = lstFind(bldHlp.cmdList, &cmd->name); - CHECK(AssertError, cmdHlp != NULL, "command help is NULL"); - - pckWriteBoolP(pack, cmd->internal); - pckWriteStrP(pack, bldHlpRenderXml(cmdHlp->summary)); - pckWriteStrP(pack, bldHlpRenderXml(cmdHlp->description)); - } - - pckWriteArrayEndP(pack); - - // Option help - // ----------------------------------------------------------------------------------------------------------------------------- - pckWriteArrayBeginP(pack); - - for (unsigned int optIdx = 0; optIdx < lstSize(bldCfg.optList); optIdx++) - { - const BldCfgOption *const opt = lstGet(bldCfg.optList, optIdx); - const BldHlpOption *const optHlp = lstFind(bldHlp.optList, &opt->name); - - // Internal - pckWriteBoolP(pack, opt->internal); - - // Section - if (optHlp != NULL) - pckWriteStrP(pack, optHlp->section); - else - pckWriteNullP(pack); - - // Summary - if (optHlp != NULL) - pckWriteStrP(pack, bldHlpRenderXml(optHlp->summary)); - else - pckWriteNullP(pack); - - // Description - if (optHlp != NULL) - pckWriteStrP(pack, bldHlpRenderXml(optHlp->description)); - else - pckWriteNullP(pack); - - // Deprecations - StringList *const deprecateList = strLstNew(); - - if (opt->deprecateList != NULL) - { - for (unsigned int deprecateIdx = 0; deprecateIdx < lstSize(opt->deprecateList); deprecateIdx++) - { - const BldCfgOptionDeprecate *const deprecate = lstGet(opt->deprecateList, deprecateIdx); - - if (!strEq(deprecate->name, opt->name)) - strLstAdd(deprecateList, deprecate->name); - } - } - - if (!strLstEmpty(deprecateList)) - { - pckWriteArrayBeginP(pack); - - for (unsigned int deprecateIdx = 0; deprecateIdx < strLstSize(deprecateList); deprecateIdx++) - pckWriteStrP(pack, strLstGet(deprecateList, deprecateIdx)); - - pckWriteArrayEndP(pack); - } - else - pckWriteNullP(pack); - - // Command overrides - bool found = false; - - for (unsigned int cmdIdx = 0; cmdIdx < lstSize(bldCfg.cmdList); cmdIdx++) - { - const BldCfgCommand *const cmd = lstGet(bldCfg.cmdList, cmdIdx); - const BldCfgCommand *const optCmd = lstFind(opt->cmdList, &cmd->name); - - if (optCmd != NULL) - { - const BldHlpCommand *const cmdHlp = lstFind(bldHlp.cmdList, &cmd->name); - CHECK(AssertError, cmdHlp != NULL, "command help is NULL"); - const BldHlpOption *const cmdOptHlp = cmdHlp->optList != NULL ? lstFind(cmdHlp->optList, &opt->name) : NULL; - - if (opt->internal != optCmd->internal || cmdOptHlp != NULL) - { - if (!found) - { - pckWriteArrayBeginP(pack); - found = true; - } - - pckWriteObjBeginP(pack, .id = cmdIdx + 1); - - if (opt->internal != optCmd->internal) - pckWriteBoolP(pack, optCmd->internal, .defaultWrite = true); - else - pckWriteNullP(pack); - - if (cmdOptHlp != NULL) - { - pckWriteStrP(pack, bldHlpRenderXml(cmdOptHlp->summary)); - pckWriteStrP(pack, bldHlpRenderXml(cmdOptHlp->description)); - } - - pckWriteObjEndP(pack); - } - } - } - - if (found) - pckWriteArrayEndP(pack); - else - pckWriteNullP(pack); - } - - pckWriteArrayEndP(pack); - pckWriteEnd(pack); - - return pack; -} - -/*********************************************************************************************************************************** -Compress pack to a buffer -***********************************************************************************************************************************/ -static Buffer * -bldHlpRenderHelpAutoCCmp(const BldCfg bldCfg, const BldHlp bldHlp) -{ - // Get pack buffer - const Buffer *const packBuf = pckToBuf(pckWriteResult(bldHlpRenderHelpAutoCPack(bldCfg, bldHlp))); - Buffer *const result = bufNew(bufSize(packBuf)); - - // Open source/destination - IoRead *const source = ioBufferReadNewOpen(packBuf); - IoWrite *const destination = ioBufferWriteNew(result); - - ioFilterGroupAdd(ioWriteFilterGroup(destination), bz2CompressNew(9, false)); - ioWriteOpen(destination); - - // Copy data from source to destination - Buffer *read = bufNew(bufUsed(packBuf) + 1); - - ioRead(source, read); - ASSERT(ioReadEof(source)); - - ioWrite(destination, read); - ioWriteClose(destination); - - // Return compressed buffer - return result; -} - -/*********************************************************************************************************************************** -Output buffer to a file as a C string -***********************************************************************************************************************************/ -// Map bytes that have a short named escape to their two-character escape sequence. Bytes not in the map are written directly when -// printable or as a three-digit octal escape otherwise. -static const char *const helpDataEscapeMap[256] = -{ - ['\a'] = "\\a", ['\b'] = "\\b", ['\t'] = "\\t", ['\n'] = "\\n", ['\v'] = "\\v", ['\f'] = "\\f", ['\r'] = "\\r", - ['"'] = "\\\"", ['\\'] = "\\\\", ['?'] = "\\?", -}; - -static void -bldHlpRenderHelpAutoC(const Storage *const storageRepo, const BldCfg bldCfg, const BldHlp bldHlp) -{ - // Convert buffer to bytes - const Buffer *const buffer = bldHlpRenderHelpAutoCCmp(bldCfg, bldHlp); - - String *const help = strCatFmt( - strNew(), - "%s" - "VR_NON_STRING static const char helpData[%zu] =\n" - "\"", - strZ(bldHeader("help", "Help Data")), bufUsed(buffer)); - - // Encode the compressed data as a single C string literal. This is much more compact than a list of byte values, which matters - // now that help.auto.c.inc is checked in as an asset. The data is not NUL-terminated so the array is declared VR_NON_STRING and - // sized to exactly match the data. - // - // Physical lines are broken at a fixed width using a backslash-newline continuation, which the preprocessor removes before the - // string is tokenized. This keeps every line the same width even when the break falls in the middle of an escape sequence. The - // opening quote occupies the first column of the first line. - size_t lineSize = 1; - - for (unsigned int bufferIdx = 0; bufferIdx < bufUsed(buffer); bufferIdx++) - { - const uint8_t byte = bufPtrConst(buffer)[bufferIdx]; - - // Look up the escape sequence, falling back to a direct printable character or a three-digit octal escape - char escBuf[5]; - const char *escZ = helpDataEscapeMap[byte]; - - if (escZ == NULL) - { - if (byte >= 0x20 && byte <= 0x7e) - { - escBuf[0] = (char)byte; - escBuf[1] = '\0'; - } - else - snprintf(escBuf, sizeof(escBuf), "\\%03o", byte); - - escZ = escBuf; - } - - // Append the escape, breaking the line with a continuation once it is full. Counting the trailing continuation backslash - // the line is 132 characters wide, matching the rest of the source. - for (; *escZ != '\0'; escZ++) - { - if (lineSize == 131) - { - strCatZ(help, "\\\n"); - lineSize = 0; - } - - strCatChr(help, *escZ); - lineSize++; - } - } - - strCatZ(help, "\";\n"); - - // Write to storage - bldPut(storageRepo, "src/command/help/help.auto.c.inc", BUFSTR(help)); -} - -/**********************************************************************************************************************************/ -void -bldHlpRender(const Storage *const storageRepo, const BldCfg bldCfg, const BldHlp bldHlp) -{ - bldHlpRenderHelpAutoC(storageRepo, bldCfg, bldHlp); -} diff --git a/src/build/help/render.h b/src/build/help/render.h deleted file mode 100644 index bd482f839..000000000 --- a/src/build/help/render.h +++ /dev/null @@ -1,19 +0,0 @@ -/*********************************************************************************************************************************** -Render Help -***********************************************************************************************************************************/ -#ifndef BUILD_HELP_RENDER_H -#define BUILD_HELP_RENDER_H - -#include "build/config/parse.h" -#include "build/help/parse.h" - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Render help -void bldHlpRender(const Storage *const storageRepo, const BldCfg bldCfg, const BldHlp bldHlp); - -// Render xml as text -String *bldHlpRenderXml(const XmlNode *xml); - -#endif diff --git a/src/build/main.c b/src/build/main.c deleted file mode 100644 index ac0ff9757..000000000 --- a/src/build/main.c +++ /dev/null @@ -1,110 +0,0 @@ -/*********************************************************************************************************************************** -Code Builder -***********************************************************************************************************************************/ -#include - -#include - -#include "common/log.h" -#include "storage/posix/storage.h" - -#include "build/config/parse.h" -#include "build/config/render.h" -#include "build/error/parse.h" -#include "build/error/render.h" -#include "build/help/parse.h" -#include "build/help/render.h" -#include "build/postgres/parse.h" -#include "build/postgres/render.h" - -int -main(const int argListSize, const char *const argList[]) -{ - // Set stack trace and mem context error cleanup handlers - static const ErrorHandlerFunction errorHandlerList[] = {stackTraceClean, memContextClean}; - errorHandlerSet(errorHandlerList, LENGTH_OF(errorHandlerList)); - - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(INT, argListSize); - FUNCTION_LOG_PARAM(CHARPY, argList); - FUNCTION_LOG_END(); - - int result = 0; - - TRY_BEGIN() - { - // Check parameters - CHECK(ParamInvalidError, argListSize >= 2 && argListSize <= 4, "only one to three parameters allowed"); - - // Initialize logging - logInit(logLevelWarn, logLevelError, logLevelOff, false, 0, 1, false); - - // Get current working directory - char currentWorkDir[1024]; - THROW_ON_SYS_ERROR(getcwd(currentWorkDir, sizeof(currentWorkDir)) == NULL, FormatError, "unable to get cwd"); - - // Get repo path (cwd if it was not passed) - const String *pathRepo = strPath(STR(currentWorkDir)); - String *pathBuild = strCat(strNew(), pathRepo); - - if (argListSize >= 3) - { - const String *const pathArg = STR(argList[2]); - - if (strBeginsWith(pathArg, FSLASH_STR)) - pathRepo = strPath(pathArg); - else - pathRepo = strPathAbsolute(pathArg, STR(currentWorkDir)); - - pathBuild = strDup(pathRepo); - } - - // If the build path was specified - if (argListSize >= 4) - { - const String *const pathArg = STR(argList[3]); - - if (strBeginsWith(pathArg, FSLASH_STR)) - pathBuild = strDup(pathArg); - else - pathBuild = strPathAbsolute(pathArg, STR(currentWorkDir)); - } - - // Repo and build storage - const Storage *const storageRepo = storagePosixNewP(pathRepo); - const Storage *const storageBuild = storagePosixNewP(pathBuild, .write = true); - - // Config - if (strEqZ(STRDEF("config"), argList[1])) - bldCfgRender(storageBuild, bldCfgParse(storageRepo), true); - - // Error - if (strEqZ(STRDEF("error"), argList[1])) - bldErrRender(storageBuild, bldErrParse(storageRepo)); - - // Help - if (strEqZ(STRDEF("help"), argList[1])) - { - const BldCfg bldCfg = bldCfgParse(storageRepo); - bldHlpRender(storageBuild, bldCfg, bldHlpParse(storageRepo, bldCfg, false)); - } - - // PostgreSQL - if (strEqZ(STRDEF("postgres"), argList[1])) - bldPgRender(storageBuild, bldPgParse(storageRepo)); - - if (strEqZ(STRDEF("postgres-version"), argList[1])) - bldPgVersionRender(storageBuild, bldPgParse(storageRepo)); - } - CATCH_FATAL() - { - LOG_FMT( - errorCode() == errorTypeCode(&AssertError) ? logLevelAssert : logLevelError, errorCode(), "%s\n%s", errorMessage(), - errorStackTrace()); - - result = errorCode(); - } - TRY_END(); - - FUNCTION_LOG_RETURN(INT, result); -} diff --git a/src/build/postgres/parse.c b/src/build/postgres/parse.c deleted file mode 100644 index 93bee1585..000000000 --- a/src/build/postgres/parse.c +++ /dev/null @@ -1,197 +0,0 @@ -/*********************************************************************************************************************************** -Parse PostgreSQL Interface Yaml -***********************************************************************************************************************************/ -#include - -#include "storage/posix/storage.h" - -#include "build/common/yaml.h" -#include "build/postgres/parse.h" - -/*********************************************************************************************************************************** -Parse version list -***********************************************************************************************************************************/ -typedef struct BldPgVersionRaw -{ - const String *version; // See BldPgVersion for comments - bool release; -} BldPgVersionRaw; - -static List * -bldPgVersionList(Yaml *const yaml) -{ - List *const result = lstNewP(sizeof(BldPgVersion), .comparator = lstComparatorStr); - - MEM_CONTEXT_TEMP_BEGIN() - { - YamlEvent pgDef = yamlEventNextCheck(yaml, yamlEventTypeScalar); - - // Parse version list - if (strEqZ(pgDef.value, "version")) - { - YAML_SEQ_BEGIN(yaml) - { - BldPgVersionRaw pgRaw = {.release = true}; - - // If map then the version has attributes - if (yamlEventPeekIs(yaml, yamlEventTypeMapBegin)) - { - YAML_MAP_BEGIN(yaml) - { - pgRaw.version = yamlScalarNext(yaml).value; - - YAML_MAP_BEGIN(yaml) - { - const String *const verDef = yamlScalarNext(yaml).value; - const YamlEvent verDefVal = yamlScalarNext(yaml); - - // Get release setting - if (strEqZ(verDef, "release")) - pgRaw.release = yamlBoolParse(verDefVal); - else - THROW_FMT(FormatError, "unknown postgres definition '%s'", strZ(verDef)); - } - YAML_MAP_END(); - } - YAML_MAP_END(); - } - // Else the version is a scalar - else - pgRaw.version = yamlScalarNext(yaml).value; - - // Add to list - MEM_CONTEXT_BEGIN(lstMemContext(result)) - { - const BldPgVersion bldPgVersion = - { - .version = strDup(pgRaw.version), - .release = pgRaw.release - }; - - lstAdd(result, &bldPgVersion); - } - MEM_CONTEXT_END(); - } - YAML_SEQ_END(); - } - else - THROW_FMT(FormatError, "unknown postgres definition '%s'", strZ(pgDef.value)); - } - MEM_CONTEXT_TEMP_END(); - - return result; -} - -/*********************************************************************************************************************************** -Parse defines from header -***********************************************************************************************************************************/ -static StringList * -bldPgParseDefine(const String *const header) -{ - const StringList *const lineList = strLstNewSplitZ(header, "\n"); - StringList *const result = strLstNew(); - - // Scan all lines - for (unsigned int lineIdx = 0; lineIdx < strLstSize(lineList); lineIdx++) - { - const String *const line = strTrim(strLstGet(lineList, lineIdx)); - - // If define get name - if (strBeginsWithZ(line, "#define")) - { - const String *const defineToken = strTrim(strLstGet(strLstNewSplitZ(line, " "), 1)); - - if (strEmpty(defineToken)) - THROW_FMT(FormatError, "unable to find define -- are there extra spaces on '%s'", strZ(line)); - - // The define might be followed by a ( or tab - const StringList *defineList = strLstNewSplitZ(defineToken, "("); - const String *define; - - if (strLstSize(defineList) > 1) - define = strTrim(strLstGet(defineList, 0)); - else - { - defineList = strLstNewSplitZ(defineToken, "\t"); - define = strTrim(strLstGet(defineList, 0)); - } - - strLstAddIfMissing(result, define); - } - } - - return result; -} - -/*********************************************************************************************************************************** -Parse types from header -***********************************************************************************************************************************/ -static StringList * -bldPgParseType(const String *const header) -{ - const StringList *const lineList = strLstNewSplitZ(header, "\n"); - StringList *const result = strLstNew(); - bool scanEnum = false; - - // Scan all lines - for (unsigned int lineIdx = 0; lineIdx < strLstSize(lineList); lineIdx++) - { - const String *const line = strTrim(strLstGet(lineList, lineIdx)); - const StringList *const tokenList = strLstNewSplitZ(line, " "); - const String *const tokenFirst = strLstGet(tokenList, 0); - - // If typedef - if (strEqZ(tokenFirst, "typedef")) - { - const String *const tokenType = strLstGet(tokenList, 1); - - // If struct/enum continue scanning to get the name/enums - if (strEqZ(tokenType, "struct") || strEqZ(tokenType, "enum")) - { - scanEnum = strEqZ(tokenType, "enum"); - } - // Else add the type name - else - strLstAddIfMissing(result, strLstGet(strLstNewSplitZ(strLstGet(tokenList, strLstSize(tokenList) - 1), ";"), 0)); - } - // End scanning of struct/enum and get name - else if (strEqZ(tokenFirst, "}")) - { - strLstAddIfMissing(result, strLstGet(strLstNewSplitZ(strLstGet(tokenList, strLstSize(tokenList) - 1), ";"), 0)); - scanEnum = false; - } - // Add enums to type list - else if (scanEnum && !strEqZ(tokenFirst, "{")) - strLstAddIfMissing(result, strLstGet(strLstNewSplitZ(strLstGet(tokenList, 0), ","), 0)); - } - - return result; -} - -/**********************************************************************************************************************************/ -BldPg -bldPgParse(const Storage *const storageRepo) -{ - // Parse types from version.vendor.h - const String *const vendorHeader = strNewBuf( - storageGetP(storageNewReadP(storageRepo, STRDEF("src/postgres/interface/version.vendor.h")))); - StringList *const typeList = bldPgParseType(vendorHeader); - strLstSort(typeList, sortOrderAsc); - - // Parse defines from version.vendor.h - StringList *const defineList = bldPgParseDefine(vendorHeader); - strLstAddZ(defineList, "CATALOG_VERSION_NO_MAX"); - strLstAddZ(defineList, "PG_VERSION"); - strLstSort(defineList, sortOrderAsc); - - // Parse defines from version.intern.h - const StringList *const functionList = bldPgParseDefine( - strNewBuf(storageGetP(storageNewReadP(storageRepo, STRDEF("src/postgres/interface/version.intern.h"))))); - - // Initialize yaml - Yaml *const yaml = yamlNew(storageGetP(storageNewReadP(storageRepo, STRDEF("src/build/postgres/postgres.yaml")))); - yamlEventNextCheck(yaml, yamlEventTypeMapBegin); - - // Parse postgres - return (BldPg){.pgList = bldPgVersionList(yaml), .typeList = typeList, .defineList = defineList, .functionList = functionList}; -} diff --git a/src/build/postgres/parse.h b/src/build/postgres/parse.h deleted file mode 100644 index d21a1fba4..000000000 --- a/src/build/postgres/parse.h +++ /dev/null @@ -1,32 +0,0 @@ -/*********************************************************************************************************************************** -Parse PostgreSQL Interface Yaml -***********************************************************************************************************************************/ -#ifndef BUILD_POSTRES_PARSE_H -#define BUILD_POSTRES_PARSE_H - -#include "build/common/string.h" - -/*********************************************************************************************************************************** -Types -***********************************************************************************************************************************/ -typedef struct BldPgVersion -{ - const String *version; // Version - bool release; // Is this a released version? -} BldPgVersion; - -typedef struct BldPg -{ - const List *pgList; // Supported PostgreSQL versions - const StringList *typeList; // PostgreSQL interface types - const StringList *defineList; // PostgreSQL interface defines - const StringList *functionList; // Functions defined by macros -} BldPg; - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Parse postgres.yaml -BldPg bldPgParse(const Storage *const storageRepo); - -#endif diff --git a/src/build/postgres/render.c b/src/build/postgres/render.c deleted file mode 100644 index 2a04511e4..000000000 --- a/src/build/postgres/render.c +++ /dev/null @@ -1,228 +0,0 @@ -/*********************************************************************************************************************************** -Render PostgreSQL Interface -***********************************************************************************************************************************/ -#include - -#include "storage/posix/storage.h" - -#include "build/common/render.h" -#include "build/postgres/render.h" - -/*********************************************************************************************************************************** -Render interface.auto.c.inc -***********************************************************************************************************************************/ -#define PG_MODULE "postgres" -#define PG_AUTO_COMMENT "PostgreSQL Interface" - -static void -bldPgRenderInterfaceAutoC(const Storage *const storageRepo, const BldPg bldPg) -{ - String *pg = bldHeader(PG_MODULE, PG_AUTO_COMMENT); - - // PostgreSQL interfaces - // ----------------------------------------------------------------------------------------------------------------------------- - for (unsigned int pgIdx = lstSize(bldPg.pgList) - 1; pgIdx < lstSize(bldPg.pgList); pgIdx--) - { - const BldPgVersion *const pgVersion = lstGet(bldPg.pgList, pgIdx); - const char *const versionNoDot = strZ(strLstJoin(strLstNewSplitZ(pgVersion->version, "."), "")); - - strCatFmt( - pg, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "PostgreSQL %s interface\n" - COMMENT_BLOCK_END "\n" - "#define PG_VERSION PG_VERSION_%s\n" - "\n", - strZ(pgVersion->version), versionNoDot); - - for (unsigned int typeIdx = 0; typeIdx < strLstSize(bldPg.typeList); typeIdx++) - { - const String *const type = strLstGet(bldPg.typeList, typeIdx); - - strCat(pg, bldDefineRender(type, strNewFmt("%s_%s", strZ(type), versionNoDot))); - strCatChr(pg, '\n'); - } - - if (!pgVersion->release) - { - strCatZ( - pg, - "\n" - "#define CATALOG_VERSION_NO_MAX\n"); - } - - strCatZ( - pg, - "\n" - "#include \"postgres/interface/version.intern.h\"\n" - "\n"); - - for (unsigned int functionIdx = 0; functionIdx < strLstSize(bldPg.functionList); functionIdx++) - strCatFmt(pg, "%s(%s);\n", strZ(strLstGet(bldPg.functionList, functionIdx)), versionNoDot); - - strCatChr(pg, '\n'); - - for (unsigned int typeIdx = 0; typeIdx < strLstSize(bldPg.typeList); typeIdx++) - strCatFmt(pg, "#undef %s\n", strZ(strLstGet(bldPg.typeList, typeIdx))); - - strCatChr(pg, '\n'); - - for (unsigned int defineIdx = 0; defineIdx < strLstSize(bldPg.defineList); defineIdx++) - strCatFmt(pg, "#undef %s\n", strZ(strLstGet(bldPg.defineList, defineIdx))); - - strCatChr(pg, '\n'); - - for (unsigned int functionIdx = 0; functionIdx < strLstSize(bldPg.functionList); functionIdx++) - strCatFmt(pg, "#undef %s\n", strZ(strLstGet(bldPg.functionList, functionIdx))); - } - - // Interface struct - // ----------------------------------------------------------------------------------------------------------------------------- - strCatFmt( - pg, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "PostgreSQL interface struct\n" - COMMENT_BLOCK_END "\n" - "static const PgInterface pgInterface[] =\n" - "{\n"); - - for (unsigned int pgIdx = lstSize(bldPg.pgList) - 1; pgIdx < lstSize(bldPg.pgList); pgIdx--) - { - const BldPgVersion *const pgVersion = lstGet(bldPg.pgList, pgIdx); - const char *const versionNoDot = strZ(strLstJoin(strLstNewSplitZ(pgVersion->version, "."), "")); - - strCatFmt( - pg, - " {\n" - " .version = PG_VERSION_%s,\n" - "\n", - versionNoDot); - - for (unsigned int functionIdx = 0; functionIdx < strLstSize(bldPg.functionList); functionIdx++) - { - // Convert define name to function name - const StringList *const nameList = strLstNewSplitZ(strLstGet(bldPg.functionList, functionIdx), "_"); - String *const name = strNew(); - - for (unsigned int nameIdx = 0; nameIdx < strLstSize(nameList); nameIdx++) - { - String *const namePart = strLower(strLstGet(nameList, nameIdx)); - - if (nameIdx != 0) - strFirstUpper(namePart); - - strCat(name, namePart); - } - - strCatFmt( - pg, - " .%s = %s%s,\n", - strZ(strFirstLower(strSub(name, sizeof("pgInterface") - 1))), strZ(name), versionNoDot); - } - - strCatZ( - pg, - " },\n"); - } - - strCatFmt( - pg, - "};\n"); - - bldPut(storageRepo, "src/postgres/interface.auto.c.inc", BUFSTR(pg)); -} - -/**********************************************************************************************************************************/ -void -bldPgRender(const Storage *const storageRepo, const BldPg bldPg) -{ - bldPgRenderInterfaceAutoC(storageRepo, bldPg); -} - -/*********************************************************************************************************************************** -Render version.auto.h -***********************************************************************************************************************************/ -#define PG_VERSION_MODULE "postgres-version" -#define PG_VERSION_AUTO_COMMENT "PostgreSQL Version" - -static void -bldPgRenderVersionAutoH(const Storage *const storageRepo, const BldPg bldPg) -{ - String *const pg = strCatFmt( - strNew(), - "%s" - "#ifndef POSTGRES_VERSION_AUTO_H\n" - "#define POSTGRES_VERSION_AUTO_H\n", - strZ(bldHeader(PG_VERSION_MODULE, PG_VERSION_AUTO_COMMENT))); - - // PostgreSQL version numbers - // ----------------------------------------------------------------------------------------------------------------------------- - strCatFmt( - pg, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "PostgreSQL version constants\n" - COMMENT_BLOCK_END "\n"); - - for (unsigned int pgIdx = 0; pgIdx < lstSize(bldPg.pgList); pgIdx++) - { - const BldPgVersion *const pgVersion = lstGet(bldPg.pgList, pgIdx); - const char *const versionNoDot = strZ(strLstJoin(strLstNewSplitZ(pgVersion->version, "."), "")); - - // Generate version number - unsigned int versionNum; - const int idxStart = strChr(pgVersion->version, '.'); - - if (idxStart == -1) - versionNum = cvtZToUInt(strZ(pgVersion->version)) * 10000; - else - { - versionNum = - cvtZSubNToUInt(strZ(pgVersion->version), 0, (size_t)idxStart) * 10000 + - cvtZToUInt(strZ(pgVersion->version) + (size_t)idxStart + 1) * 100; - } - - // Output version number - strCatFmt(pg, "%s\n", strZ(bldDefineRender(strNewFmt("PG_VERSION_%s", versionNoDot), strNewFmt("%u", versionNum)))); - - // Output max version - if (pgIdx == lstSize(bldPg.pgList) - 1) - strCatFmt(pg, "\n%s\n", strZ(bldDefineRender(STRDEF("PG_VERSION_MAX"), strNewFmt("PG_VERSION_%s", versionNoDot)))); - } - - // PostgreSQL version strings - // ----------------------------------------------------------------------------------------------------------------------------- - strCatFmt( - pg, - "\n" - COMMENT_BLOCK_BEGIN "\n" - "PostgreSQL version string constants for use in error messages\n" - COMMENT_BLOCK_END "\n"); - - for (unsigned int pgIdx = 0; pgIdx < lstSize(bldPg.pgList); pgIdx++) - { - const BldPgVersion *const pgVersion = lstGet(bldPg.pgList, pgIdx); - const char *const versionNoDot = strZ(strLstJoin(strLstNewSplitZ(pgVersion->version, "."), "")); - - // Output version string - strCatFmt( - pg, "%s\n", - strZ(bldDefineRender(strNewFmt("PG_VERSION_%s_Z", versionNoDot), strNewFmt("\"%s\"", strZ(pgVersion->version))))); - } - - strCatZ( - pg, - "\n" - "#endif\n"); - - bldPut(storageRepo, "src/postgres/version.auto.h", BUFSTR(pg)); -} - -/**********************************************************************************************************************************/ -void -bldPgVersionRender(const Storage *const storageRepo, const BldPg bldPg) -{ - bldPgRenderVersionAutoH(storageRepo, bldPg); -} diff --git a/src/build/postgres/render.h b/src/build/postgres/render.h deleted file mode 100644 index 1f44690fd..000000000 --- a/src/build/postgres/render.h +++ /dev/null @@ -1,16 +0,0 @@ -/*********************************************************************************************************************************** -Render PostgreSQL Interface -***********************************************************************************************************************************/ -#ifndef BUILD_POSTGRES_RENDER_H -#define BUILD_POSTGRES_RENDER_H - -#include "build/postgres/parse.h" - -/*********************************************************************************************************************************** -Functions -***********************************************************************************************************************************/ -// Render auto-generated PostgreSQL files -void bldPgRender(const Storage *const storageRepo, const BldPg bldPg); -void bldPgVersionRender(const Storage *const storageRepo, const BldPg bldPg); - -#endif diff --git a/src/command/help/meson.build b/src/command/help/meson.build index e327152a2..a9ed1e0ec 100644 --- a/src/command/help/meson.build +++ b/src/command/help/meson.build @@ -7,13 +7,13 @@ if not dist 'help.auto.c.inc', output: 'help.auto.c.inc', depend_files: [ - '../../build/config/config.yaml', - '../../build/help/help.xml', + '../../../build/config.yaml', + '../../../doc/xml/reference.xml', ], command: [ build_code, 'help', - '@CURRENT_SOURCE_DIR@/../../..', + '--repo-path=@CURRENT_SOURCE_DIR@/../../..', ], ) else diff --git a/src/common/error/error.auto.c.inc b/src/common/error/error.auto.c.inc index 0998b4542..aef949395 100644 --- a/src/common/error/error.auto.c.inc +++ b/src/common/error/error.auto.c.inc @@ -1,7 +1,7 @@ /*********************************************************************************************************************************** Error Type Definition -Automatically generated by 'build-code error' -- do not modify directly. +Automatically generated by 'build.py error' -- do not modify directly. ***********************************************************************************************************************************/ /*********************************************************************************************************************************** diff --git a/src/common/error/error.auto.h b/src/common/error/error.auto.h index 08c932abb..34f546954 100644 --- a/src/common/error/error.auto.h +++ b/src/common/error/error.auto.h @@ -1,7 +1,7 @@ /*********************************************************************************************************************************** Error Type Definition -Automatically generated by 'build-code error' -- do not modify directly. +Automatically generated by 'build.py error' -- do not modify directly. ***********************************************************************************************************************************/ #ifndef COMMON_ERROR_ERROR_AUTO_H #define COMMON_ERROR_ERROR_AUTO_H diff --git a/src/config/config.auto.h b/src/config/config.auto.h index f0707def8..dde1303bf 100644 --- a/src/config/config.auto.h +++ b/src/config/config.auto.h @@ -1,7 +1,7 @@ /*********************************************************************************************************************************** Command and Option Configuration -Automatically generated by 'build-code config' -- do not modify directly. +Automatically generated by 'build.py config' -- do not modify directly. ***********************************************************************************************************************************/ #ifndef CONFIG_CONFIG_AUTO_H #define CONFIG_CONFIG_AUTO_H diff --git a/src/config/parse.auto.c.inc b/src/config/parse.auto.c.inc index 525a5d444..28bf84d17 100644 --- a/src/config/parse.auto.c.inc +++ b/src/config/parse.auto.c.inc @@ -1,7 +1,7 @@ /*********************************************************************************************************************************** Config Parse Rules -Automatically generated by 'build-code config' -- do not modify directly. +Automatically generated by 'build.py config' -- do not modify directly. ***********************************************************************************************************************************/ /*********************************************************************************************************************************** diff --git a/src/meson.build b/src/meson.build index 4daadbe55..c692dfe60 100644 --- a/src/meson.build +++ b/src/meson.build @@ -26,91 +26,12 @@ endif configure_file(output: 'build.auto.h', configuration: configuration) #################################################################################################################################### -# Common source used by all targets +# Code generation #################################################################################################################################### -src_common = files( - 'common/debug.c', - 'common/encode.c', - 'common/error/error.c', - 'common/io/filter/buffer.c', - 'common/io/filter/filter.c', - 'common/io/filter/group.c', - 'common/io/filter/sink.c', - 'common/io/bufferRead.c', - 'common/io/bufferWrite.c', - 'common/io/io.c', - 'common/io/limitRead.c', - 'common/io/read.c', - 'common/io/write.c', - 'common/log.c', - 'common/memContext.c', - 'common/stackTrace.c', - 'common/time.c', - 'common/type/blob.c', - 'common/type/buffer.c', - 'common/type/convert.c', - 'common/type/keyValue.c', - 'common/type/list.c', - 'common/type/object.c', - 'common/type/pack.c', - 'common/type/stringId.c', - 'common/type/stringList.c', - 'common/type/stringStatic.c', - 'common/type/stringZ.c', - 'common/type/variant.c', - 'common/type/variantList.c', - 'common/user.c', - 'common/wait.c', - 'config/common.c', - 'storage/posix/read.c', - 'storage/posix/storage.c', - 'storage/posix/write.c', - 'storage/iterator.c', - 'storage/list.c', - 'storage/read.c', - 'storage/readMulti.c', - 'storage/storage.c', - 'storage/write.c', -) - -#################################################################################################################################### -# Build code target -#################################################################################################################################### -src_build_code = [ - 'build/common/regExp.c', - 'build/common/render.c', - 'build/common/string.c', - 'build/common/xml.c', - 'build/common/yaml.c', - 'build/config/parse.c', - 'build/config/render.c', - 'build/error/parse.c', - 'build/error/render.c', - 'build/help/parse.c', - 'build/help/render.c', - 'build/postgres/parse.c', - 'build/postgres/render.c', - 'build/main.c', - 'common/compress/common.c', - 'common/compress/bz2/common.c', - 'common/compress/bz2/compress.c', -] - +# Generator that writes the code built from the declarations in the source. A distribution tarball ships the generated files and +# excludes the generator, and a unit test build comments out this subdir, so neither looks for it. if not dist - build_code = executable( - 'build-code', - src_common, - src_build_code, - dependencies: [ - lib_backtrace, - lib_bz2, - lib_xml, - lib_yaml - ], - c_args: [ - arg_unity, - ], - ) + build_code = find_program('../build/build.py') endif # Build help.auto.c.inc (or use the copy shipped in the distribution tarball) @@ -141,17 +62,18 @@ src_pgbackrest = [ 'command/check/check.c', 'command/check/common.c', 'command/check/report.c', - 'command/exit.c', - 'command/expire/expire.c', - 'command/help/help.c', - 'command/info/info.c', 'command/command.c', 'command/control/common.c', 'command/control/start.c', 'command/control/stop.c', + 'command/exit.c', + 'command/expire/expire.c', + 'command/help/help.c', + 'command/info/info.c', 'command/local/local.c', 'command/lock.c', 'command/manifest/manifest.c', + 'command/remote/remote.c', 'command/repo/common.c', 'command/repo/get.c', 'command/repo/ls.c', @@ -162,7 +84,6 @@ src_pgbackrest = [ 'command/restore/protocol.c', 'command/restore/restore.c', 'command/restore/timeline.c', - 'command/remote/remote.c', 'command/server/ping.c', 'command/server/server.c', 'command/stanza/common.c', @@ -172,14 +93,14 @@ src_pgbackrest = [ 'command/verify/file.c', 'command/verify/protocol.c', 'command/verify/verify.c', - 'common/compress/common.c', - 'common/compress/helper.c', 'common/compress/bz2/common.c', 'common/compress/bz2/compress.c', 'common/compress/bz2/decompress.c', + 'common/compress/common.c', 'common/compress/gz/common.c', 'common/compress/gz/compress.c', 'common/compress/gz/decompress.c', + 'common/compress/helper.c', 'common/compress/lz4/common.c', 'common/compress/lz4/compress.c', 'common/compress/lz4/decompress.c', @@ -190,14 +111,23 @@ src_pgbackrest = [ 'common/crypto/common.c', 'common/crypto/hash.c', 'common/crypto/xxhash.c', + 'common/debug.c', + 'common/encode.c', + 'common/error/error.c', 'common/error/retry.c', 'common/exec.c', 'common/fork.c', 'common/ini.c', + 'common/io/bufferRead.c', + 'common/io/bufferWrite.c', 'common/io/client.c', 'common/io/fd.c', 'common/io/fdRead.c', 'common/io/fdWrite.c', + 'common/io/filter/buffer.c', + 'common/io/filter/filter.c', + 'common/io/filter/group.c', + 'common/io/filter/sink.c', 'common/io/filter/size.c', 'common/io/http/client.c', 'common/io/http/common.c', @@ -207,6 +137,9 @@ src_pgbackrest = [ 'common/io/http/response.c', 'common/io/http/session.c', 'common/io/http/url.c', + 'common/io/io.c', + 'common/io/limitRead.c', + 'common/io/read.c', 'common/io/server.c', 'common/io/session.c', 'common/io/socket/address.c', @@ -218,12 +151,33 @@ src_pgbackrest = [ 'common/io/tls/common.c', 'common/io/tls/server.c', 'common/io/tls/session.c', + 'common/io/write.c', 'common/lock.c', + 'common/log.c', + 'common/memContext.c', 'common/regExp.c', + 'common/stackTrace.c', 'common/stat.c', + 'common/time.c', + 'common/type/blob.c', + 'common/type/buffer.c', + 'common/type/convert.c', 'common/type/json.c', + 'common/type/keyValue.c', + 'common/type/list.c', + 'common/type/object.c', + 'common/type/pack.c', 'common/type/string.c', + 'common/type/stringId.c', + 'common/type/stringList.c', + 'common/type/stringStatic.c', + 'common/type/stringZ.c', + 'common/type/variant.c', + 'common/type/variantList.c', 'common/type/xml.c', + 'common/user.c', + 'common/wait.c', + 'config/common.c', 'config/config.c', 'config/exec.c', 'config/load.c', @@ -257,9 +211,16 @@ src_pgbackrest = [ 'storage/gcs/storage.c', 'storage/gcs/write.c', 'storage/helper.c', + 'storage/iterator.c', + 'storage/list.c', + 'storage/posix/read.c', + 'storage/posix/storage.c', + 'storage/posix/write.c', + 'storage/read.c', + 'storage/readMulti.c', + 'storage/remote/protocol.c', 'storage/remote/read.c', 'storage/remote/readMulti.c', - 'storage/remote/protocol.c', 'storage/remote/storage.c', 'storage/remote/write.c', 'storage/s3/helper.c', @@ -270,12 +231,13 @@ src_pgbackrest = [ 'storage/sftp/read.c', 'storage/sftp/storage.c', 'storage/sftp/write.c', + 'storage/storage.c', + 'storage/write.c', 'main.c', ] pgbackrest_exe = executable( 'pgbackrest', - src_common, src_pgbackrest, help_auto_c_inc, interface_auto_c_inc, diff --git a/src/postgres/meson.build b/src/postgres/meson.build index aeb2a320f..de561ea12 100644 --- a/src/postgres/meson.build +++ b/src/postgres/meson.build @@ -7,12 +7,12 @@ if not dist 'interface.auto.c.inc', output: 'interface.auto.c.inc', depend_files: [ - '../build/postgres/postgres.yaml', + '../../build/postgres.yaml', ], command: [ build_code, 'postgres', - '@CURRENT_SOURCE_DIR@/../..', + '--repo-path=@CURRENT_SOURCE_DIR@/../..', ], ) else diff --git a/src/postgres/version.auto.h b/src/postgres/version.auto.h index 80c9fa471..bd707d6c6 100644 --- a/src/postgres/version.auto.h +++ b/src/postgres/version.auto.h @@ -1,7 +1,7 @@ /*********************************************************************************************************************************** PostgreSQL Version -Automatically generated by 'build-code postgres-version' -- do not modify directly. +Automatically generated by 'build.py postgres-version' -- do not modify directly. ***********************************************************************************************************************************/ #ifndef POSTGRES_VERSION_AUTO_H #define POSTGRES_VERSION_AUTO_H diff --git a/test/ci.py b/test/ci.py index 3436b7887..f44ba3a6e 100755 --- a/test/ci.py +++ b/test/ci.py @@ -24,10 +24,12 @@ sys.stderr = sys.stdout # import would be reported as an unexpected binary file. This must be set before the harness modules are imported below. sys.dont_write_bytecode = True -# The library lives beside this script. Insert it first so the harness modules are found before anything else on the path. -sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "lib")) +# Each tool keeps its library beside itself and may use the libraries below it in the hierarchy, which for the harness is all of +# them. Insert them first, lowest last, so the harness modules are found before anything else on the path. +for lib in ("build", "doc", "test"): + sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), lib, "lib")) -from common.error import EXIT_ERROR, TestError # noqa: E402 +from common.error import EXIT_ERROR, ToolError # noqa: E402 from common.exec import exec_one # noqa: E402 from common.log import * # noqa: E402 from common.user import user_name # noqa: E402 @@ -109,7 +111,7 @@ def test_run(config): vm_arch = "" if config.vm_arch is None else " --vm-arch=%s" % config.vm_arch # Packages the tests need that the test images already have - package = "gcc ccache git rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config libssh2-1-dev valgrind" + package = "gcc ccache git rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev pkg-config libssh2-1-dev valgrind" # Extra packages required when testing without containers if config.vm == VM_NONE: @@ -188,7 +190,7 @@ def main(): try: return command_run(cfg_load(sys.argv[1:], path_repo)) - except TestError as error: + except ToolError as error: log(ERROR, error) return error.status diff --git a/test/define.yaml b/test/define.yaml index 117296543..dd300af39 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -159,7 +159,7 @@ unit: feature: string coverage: - - build/common/string + - test/harness/string - common/type/string: included - common/type/stringZ - common/type/stringId @@ -212,7 +212,7 @@ unit: total: 3 coverage: - - build/common/regExp + - test/harness/regExp - common/regExp: included # ---------------------------------------------------------------------------------------------------------------------------- @@ -255,8 +255,7 @@ unit: total: 2 coverage: - - build/common/json - - common/type/json: included + - common/type/json # ---------------------------------------------------------------------------------------------------------------------------- - name: type-key-value @@ -270,8 +269,7 @@ unit: total: 1 coverage: - - build/common/xml - - common/type/xml: included + - common/type/xml # ---------------------------------------------------------------------------------------------------------------------------- - name: stat @@ -429,7 +427,7 @@ unit: total: 2 coverage: - - build/common/exec + - test/harness/exec # ---------------------------------------------------------------------------------------------------------------------------- - name: ini @@ -506,7 +504,7 @@ unit: - protocol/server include: - - build/common/exec + - test/harness/exec # ******************************************************************************************************************************** - name: config @@ -682,48 +680,121 @@ unit: # ******************************************************************************************************************************** - name: build + lang: python + vm: [none, u24, rh9] + # The build library, which every tool is built on. These are python so they are run directly rather than built, which means no + # total is needed since there are no runs to select. They are declared before the doc and test libraries so that those inherit + # them, which is the same direction the import hierarchy runs in. test: # ---------------------------------------------------------------------------------------------------------------------------- - - name: common - total: 2 + - name: common/error + + coverage: + - build/common/error + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: common/log + + coverage: + - build/common/log + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: common/storage + + coverage: + - build/common/storage + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: common/string + + coverage: + - build/common/string_id + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: common/render coverage: - build/common/render + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: common/yaml + + coverage: - build/common/yaml # ---------------------------------------------------------------------------------------------------------------------------- - - name: config - total: 1 + - name: common/pack + + coverage: + - build/common/pack + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: common/xml + + coverage: + - build/common/xml + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: config/common + + coverage: + - build/config/common + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: config/parse coverage: - build/config/parse + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: config/render + + coverage: - build/config/render # ---------------------------------------------------------------------------------------------------------------------------- - - name: error - total: 1 + - name: error/parse coverage: - build/error/parse + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: error/render + + coverage: - build/error/render # ---------------------------------------------------------------------------------------------------------------------------- - - name: help - total: 2 - - coverage: - - build/help/parse - - build/help/render - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: postgres - total: 1 + - name: postgres/parse coverage: - build/postgres/parse + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: postgres/render + + coverage: - build/postgres/render + # ---------------------------------------------------------------------------------------------------------------------------- + - name: help/parse + + coverage: + - build/help/parse + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: help/render + + coverage: + - build/help/render + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: command/main + + coverage: + - build/command/main + # ******************************************************************************************************************************** - name: info @@ -895,9 +966,6 @@ unit: - command/help/help - command/help/help.auto: noCode - include: - - build/help/render - # ---------------------------------------------------------------------------------------------------------------------------- - name: info total: 3 @@ -1028,19 +1096,47 @@ unit: # ******************************************************************************************************************************** - name: doc + lang: python + vm: [none, u24, rh9] test: + # The doc library, which sits on top of the build library. These are python so they are run directly rather than built, which + # means no total is needed since there are no runs to select. # ---------------------------------------------------------------------------------------------------------------------------- - - name: build - total: 2 + - name: command/build/var_store + + coverage: + - doc/command/build/var_store + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: command/build/eval + + coverage: + - doc/command/build/eval + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: command/build/pre + + coverage: + - doc/command/build/pre + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: command/build/reference + + coverage: + - doc/command/build/reference + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: command/build/man + + coverage: + - doc/command/build/man + + # ---------------------------------------------------------------------------------------------------------------------------- + - name: command/build/build coverage: - doc/command/build/build - - doc/command/build/eval - - doc/command/build/man - - doc/command/build/pre - - doc/command/build/reference - - doc/command/build/varStore # ******************************************************************************************************************************** # Tests for the test harness itself. These are python so they are run directly rather than built, which means no total is needed @@ -1053,42 +1149,12 @@ unit: vm: [none, u24, rh9] test: - # ---------------------------------------------------------------------------------------------------------------------------- - - name: common/error - - coverage: - - test/common/error - # ---------------------------------------------------------------------------------------------------------------------------- - name: common/exec coverage: - test/common/exec - # ---------------------------------------------------------------------------------------------------------------------------- - - name: common/log - - coverage: - - test/common/log - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: common/storage - - coverage: - - test/common/storage - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: common/string - - coverage: - - test/common/string_id - - # ---------------------------------------------------------------------------------------------------------------------------- - - name: common/render - - coverage: - - test/common/render - # ---------------------------------------------------------------------------------------------------------------------------- - name: common/user diff --git a/test/lib/command/code/format.py b/test/lib/command/code/format.py index bac642033..36f7f9255 100644 --- a/test/lib/command/code/format.py +++ b/test/lib/command/code/format.py @@ -9,7 +9,7 @@ Only version controlled files are formatted, which is what git reports here, so #################################################################################################################################### import os -from common.error import TestError +from common.error import ToolError from common.exec import exec_one from common.log import * @@ -19,9 +19,11 @@ _LINE_LENGTH = 132 # Files that are allowed to be executable. Everything else that is version controlled must not be, since a stray execute bit is # usually the result of an editor or a file system that does not preserve modes. _FILE_EXECUTABLE_LIST = ( + "build/build.py", "doc/doc.pl", + "doc/doc.py", "doc/release.pl", - "src/build/dist.sh", + "build/dist.sh", "test/ci.py", "test/smoke.py", "test/test.py", @@ -81,7 +83,7 @@ def _permission_check(path_repo, file_list): mode = os.stat(os.path.join(path_repo, file)).st_mode & 0o777 if mode & 0o111 and file not in _FILE_EXECUTABLE_LIST: - raise TestError("expected mode '%04o' for '%s' but found '%04o'" % (mode & 0o666, file, mode)) + raise ToolError("expected mode '%04o' for '%s' but found '%04o'" % (mode & 0o666, file, mode)) #################################################################################################################################### @@ -96,11 +98,11 @@ def cmd_code_format(config): try: _format_c(path_repo, file_list, config.check) _format_python(path_repo, file_list, config.check) - except TestError as error: + except ToolError as error: if not config.check: raise # The check is what CI runs, so it says how to fix what it found rather than only what it found - raise TestError("%s\nHINT: run 'test.py code-format' to format the code" % error) + raise ToolError("%s\nHINT: run 'test.py code-format' to format the code" % error) _permission_check(path_repo, file_list) diff --git a/test/lib/command/coverage/coverage.py b/test/lib/command/coverage/coverage.py index 6a6532c88..f53f174a1 100644 --- a/test/lib/command/coverage/coverage.py +++ b/test/lib/command/coverage/coverage.py @@ -7,7 +7,14 @@ warns about anything not fully covered, and writes the report.""" import json import os -from command.test.define import TEST_LANG_C, TEST_LANG_PYTHON, test_def_find, test_def_parse +from command.test.define import ( + TEST_LANG_C, + TEST_LANG_PYTHON, + test_def_find, + test_def_parse, + test_lib_path, + test_lib_split, +) from common.error import check from common.log import * from common.storage import file_read, file_write, path_list @@ -225,16 +232,14 @@ def coverage_module_file(name, lang=TEST_LANG_C): For example "common/error/error" becomes "src/common/error/error.c", and the python module "test/common/string_id" becomes "test/lib/common/string_id.py".""" - # The python the harness is written in lives in the library rather than beside the tests + # The python each tool is written in lives in its library rather than beside the tests if lang == TEST_LANG_PYTHON: - check(name.startswith("test/"), "python module '%s' must be under test/" % name) + lib, module = test_lib_split(name) - return "test/lib/" + name[len("test/") :] + ".py" + return "%s/%s.py" % (test_lib_path(lib), module) if name.startswith("test/"): result = "test/src/" + name[len("test/") :] + ".c" - elif name.startswith("doc/"): - result = "doc/src/" + name[len("doc/") :] + ".c" else: result = "src/" + name + ".c" @@ -252,26 +257,27 @@ def _coverage_list_build(module_def_list, module_name_list): """Build the file names of the code modules that should be fully covered by the tests that ran. A module is mapped to a file name here rather than by the caller since the mapping depends on the language of the test that - declared it.""" + declared it. The file is also what identifies a code module, since the same module name in another language is another file.""" - name_list = [] result = [] for module_name in module_name_list: module = test_def_find(module_def_list, module_name) for coverage in module.coverage_list: - if coverage.coverable and coverage.name not in name_list: - name_list.append(coverage.name) - result.append(coverage_module_file(coverage.name, module.lang)) + file = coverage_module_file(coverage.name, module.lang) + + if coverage.coverable and file not in result: + result.append(file) # Remove code modules that require a test that was not run for full coverage for module in module_def_list: for coverage in module.coverage_list: - if coverage.name in name_list and module.name not in module_name_list: + file = coverage_module_file(coverage.name, module.lang) + + if file in result and module.name not in module_name_list: log(WARN, "module '%s' did not have all tests run required for coverage" % coverage.name) - result.pop(name_list.index(coverage.name)) - name_list.remove(coverage.name) + result.remove(file) return result diff --git a/test/lib/command/lint/lint.py b/test/lib/command/lint/lint.py index bd2451e78..ccd20e9c5 100644 --- a/test/lib/command/lint/lint.py +++ b/test/lib/command/lint/lint.py @@ -5,10 +5,11 @@ claim to.""" #################################################################################################################################### import os +import re from command.lint.ascii import lint_ascii from command.lint.string_id import lint_str_id -from common.error import TestError +from common.error import ToolError from common.log import * from common.storage import path_list_recurse @@ -20,6 +21,9 @@ _FILE_SKIP_LIST = ( "doc/resource/git-history.cache", # Generated from git history, contains non-ASCII author names ) +# A python module in a tool library, e.g. build/lib/common/render.py +_LIB_MODULE_EXP = re.compile(r"^([^/]+)/lib/(.+\.py)$") + #################################################################################################################################### def _lint_str_id_apply(name): @@ -37,7 +41,21 @@ def _lint_str_id_apply(name): def cmd_lint(path_repo): """Lint every file in the repository.""" + lib_module = {} + for name in path_list_recurse(path_repo): + # A module may appear in only one library. Python resolves a module name to the first library on the path that has it and + # ignores the rest, so a duplicate would hide the shadowed module from every tool with no indication that it had. + match = _LIB_MODULE_EXP.match(name) + + if match is not None: + lib, module = match.groups() + + if module in lib_module: + raise ToolError("module '%s' is in the %s and %s libraries" % (module, lib_module[module], lib)) + + lib_module[module] = lib + # Skip files that are exempt from the content checks if name in _FILE_SKIP_LIST: continue @@ -62,4 +80,4 @@ def cmd_lint(path_repo): error_total += lint_str_id(data.decode("utf-8", errors="replace")) if error_total > 0: - raise TestError("%u linter error(s) in '%s' (see warnings above)" % (error_total, name)) + raise ToolError("%u linter error(s) in '%s' (see warnings above)" % (error_total, name)) diff --git a/test/lib/command/lint/string_id.py b/test/lib/command/lint/string_id.py index c7e875042..d42d12465 100644 --- a/test/lib/command/lint/string_id.py +++ b/test/lib/command/lint/string_id.py @@ -6,7 +6,7 @@ can be used in a switch, which means a hand-edited string and value can drift ap #################################################################################################################################### import re -from common.error import TestError +from common.error import ToolError from common.log import * from common.render import bld_str_id_seq @@ -58,7 +58,7 @@ def lint_str_id(source): expected = bld_str_id_seq(param[1:-1]) else: expected = bld_str_id_seq(param[1:-1], int(param_list[1])) - except TestError as error: + except ToolError as error: log(WARN, "'%s' is not valid: %s" % (text, error)) result += 1 diff --git a/test/lib/command/test/build.py b/test/lib/command/test/build.py index 5e1886b57..14a7a6ff2 100644 --- a/test/lib/command/test/build.py +++ b/test/lib/command/test/build.py @@ -27,9 +27,6 @@ def path_module(module_name): if module_name.startswith("test/"): return "test/src" + module_name[len("test") :] - if module_name.startswith("doc/"): - return "doc/src" + module_name[len("doc") :] - return "src/" + module_name @@ -350,7 +347,6 @@ class TestBuild: + " include_directories(\n" + " '.',\n" + " '%s/src',\n" % path_repo_rel - + " '%s/doc/src',\n" % path_repo_rel + " '%s/test/src',\n" % path_repo_rel + " ),\n" + " dependencies: [\n" @@ -366,7 +362,6 @@ class TestBuild: + " lib_pq,\n" + " lib_ssh2,\n" + " lib_xml,\n" - + " lib_yaml,\n" + " lib_z,\n" + " lib_zstd,\n" + " ],\n" diff --git a/test/lib/command/test/container.py b/test/lib/command/test/container.py index b4a508ceb..f8192b0e3 100644 --- a/test/lib/command/test/container.py +++ b/test/lib/command/test/container.py @@ -193,9 +193,9 @@ def _script_package(name, vm): result += ( " yum -y update && \\\n" - + " yum -y install openssh-server openssh-clients sudo git ca-certificates gcc make ccache meson \\\n" - + " openssl openssl-devel libyaml-devel zlib-devel libxml2-devel lz4-devel lz4 bzip2-devel bzip2 \\\n" - + " libssh2-devel zstd libzstd-devel systemd-devel" + + " yum -y install openssh-server openssh-clients sudo git ca-certificates gcc make ccache meson openssl\\\n" + + " openssl-devel zlib-devel libxml2-devel lz4-devel lz4 bzip2-devel bzip2 libssh2-devel zstd libzstd-devel\\\n" + + " systemd-devel" ) if name != VM_RH8: @@ -220,7 +220,7 @@ def _script_package(name, vm): + " apt-get update && \\\n" + " apt-get install -y --no-install-recommends openssh-server sudo gcc make git \\\n" + " ca-certificates libssl-dev tzdata zlib1g-dev libxml2-dev pkg-config \\\n" - + " libbz2-dev bzip2 libyaml-dev liblz4-dev liblz4-tool gnupg lsb-release ccache meson \\\n" + + " libbz2-dev bzip2 liblz4-dev liblz4-tool gnupg lsb-release ccache meson \\\n" + " libssh2-1-dev libcurl4-openssl-dev libsystemd-dev python3-yaml" ) @@ -238,7 +238,7 @@ def _script_package(name, vm): + " apk add --no-cache sudo openssh git rsync tzdata openssh ca-certificates openrc bash && \\\n" + " rc-update add sshd && \\\n" + " apk add --no-cache meson build-base libpq-dev openssl-dev libxml2-dev pkgconfig lz4-dev bzip2-dev\\\n" - + " openssh-keygen zlib-dev yaml-dev libssh2-dev valgrind lz4 zstd zstd-dev py3-yaml" + + " openssh-keygen zlib-dev libssh2-dev valgrind lz4 zstd zstd-dev py3-yaml" ) # Coverage for python tests where the packaged version predates the per line branch detail the report needs, so it comes from diff --git a/test/lib/command/test/define.py b/test/lib/command/test/define.py index 31d3a1686..2da93a2b4 100644 --- a/test/lib/command/test/define.py +++ b/test/lib/command/test/define.py @@ -10,7 +10,7 @@ import os import yaml -from common.error import TestError, check +from common.error import ToolError, check # Test types, in the order they are declared in define.yaml and accumulated TEST_TYPE_UNIT = "unit" @@ -25,6 +25,36 @@ TEST_LANG_PYTHON = "python" TEST_LANG_LIST = (TEST_LANG_C, TEST_LANG_PYTHON) +# Libraries a python module may import from, keyed by the library it lives in, which is the first component of its module name. A +# tool sees its own library and the ones below it in the hierarchy, listed in search order, which is what include_directories does +# for the C. +TEST_LIB_LIST = { + "build": ("build",), + "doc": ("doc", "build"), + "test": ("test", "doc", "build"), +} + + +#################################################################################################################################### +def test_lib_split(name): + """Split a python module name into the library it lives in and the module within it. + + For example "build/common/render" becomes ("build", "common/render"), where the module is imported as common.render from the + build library.""" + + lib, _, module = name.partition("/") + + check(lib in TEST_LIB_LIST, "python module '%s' must be in one of these libraries: %s" % (name, ", ".join(TEST_LIB_LIST))) + + return lib, module + + +#################################################################################################################################### +def test_lib_path(lib): + """Path of a library relative to the repository, e.g. "build" becomes "build/lib".""" + + return "%s/lib" % lib + #################################################################################################################################### class TestDefCoverage: @@ -315,4 +345,4 @@ def test_def_find(module_list, name): if module.name == name: return module - raise TestError("'%s' is not a valid test" % name) + raise ToolError("'%s' is not a valid test" % name) diff --git a/test/lib/command/test/job.py b/test/lib/command/test/job.py index 5d6d6ef95..1b834448f 100644 --- a/test/lib/command/test/job.py +++ b/test/lib/command/test/job.py @@ -278,12 +278,18 @@ class TestJob: print("") # Write the profile and coverage a C test left behind. A python test wrote its own coverage while it ran. - if status == 0 and self.run.module.lang == TEST_LANG_C and not self.run.integration: - if config.profile: - self._profile() + # + # The cleanup below runs whatever happens here, since a test leaves files behind that only the user it ran as can remove and + # this is the last chance to remove them. Left behind, they make every later run fail on a test path that cannot be cleaned. + try: + if status == 0 and self.run.module.lang == TEST_LANG_C and not self.run.integration: + if config.profile: + self._profile() - if self.coverage: - self._coverage() + if self.coverage: + self._coverage() + finally: + self._cleanup() elapsed = math.ceil((time.time() - self.time_begin) * 100) / 100 @@ -307,12 +313,18 @@ class TestJob: + (":\n\n%s\n" % self.exec.output.strip() if config.vm_out and not self.show_output else ""), ) - if config.cleanup: - # An integration test starts its own containers and names them after the job, so those are removed here too. Anything - # still running in one would keep writing to the test path while it is being removed. - if self.run.vm != VM_NONE: - container_remove("^%s($|-)" % self.container) - - exec_one("chmod -R 700 %s/* 2>&1;rm -rf %s" % (self.path_host, self.path_host)) - return True, fail + + ################################################################################################################################ + def _cleanup(self): + """Remove the containers the test ran in and the path it ran in.""" + + if not self.config.cleanup: + return + + # An integration test starts its own containers and names them after the job, so those are removed here too. Anything still + # running in one would keep writing to the test path while it is being removed. + if self.run.vm != VM_NONE: + container_remove("^%s($|-)" % self.container) + + exec_one("chmod -R 700 %s/* 2>&1;rm -rf %s" % (self.path_host, self.path_host)) diff --git a/test/lib/command/test/python.py b/test/lib/command/test/python.py index cf7103897..e8b8d9761 100644 --- a/test/lib/command/test/python.py +++ b/test/lib/command/test/python.py @@ -2,7 +2,10 @@ Runs a python test module with only the library modules it declared in define.yaml importable. A C unit test compiles in just the modules it declares, so reaching for anything else fails the build and every cross-dependency ends up documented. Python has no link -step and would happily import anything under test/lib, so an import hook reproduces that. +step and would happily import anything under a library, so an import hook reproduces that. + +Only the libraries the test module's own library sits on top of are searched, so a test of the build library cannot reach the doc or +test library any more than the build tool itself can. This runs as a script rather than inside the harness because the harness has already imported most of the library, and an import that is already in sys.modules never reaches a hook.""" @@ -13,20 +16,27 @@ import importlib.util import os import sys +# Library the harness itself lives in, which is the one this script is part of +PATH_HARNESS = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + #################################################################################################################################### class ImportGuard: """Refuse to import a library module the test did not declare.""" - def __init__(self, path_lib, allow_list): - self.path_lib = path_lib + def __init__(self, path_lib_list, allow_list): + # Libraries a module may be found in. The harness library is included even when the test may not import from it, so that + # reaching into it is refused here rather than quietly resolving on the path. + self.path_lib_list = list(path_lib_list) + [PATH_HARNESS] # The test harness is support for the tests rather than part of the harness under test, so it is always available self.allow_list = set(allow_list) | {"harness.test"} - # Top level packages of the library, i.e. the imports that are guarded. Anything else, e.g. the standard library, is left - # to the finders that follow this one. - self.root_list = {name for name in os.listdir(path_lib) if os.path.isdir(os.path.join(path_lib, name))} + # Top level packages of every library, i.e. the imports that are guarded. Anything else, e.g. the standard library, is left to + # the finders that follow this one. + self.root_list = { + name for path_lib in self.path_lib_list for name in os.listdir(path_lib) if os.path.isdir(os.path.join(path_lib, name)) + } ################################################################################################################################ def find_spec(self, fullname, path=None, target=None): @@ -36,8 +46,9 @@ class ImportGuard: return None # A package holds no code of its own, and a module is reached through the packages above it, so only modules are declared - if os.path.isdir(os.path.join(self.path_lib, fullname.replace(".", os.sep))): - return None + for path_lib in self.path_lib_list: + if os.path.isdir(os.path.join(path_lib, fullname.replace(".", os.sep))): + return None raise ImportError( "'%s' is not declared by this test module -- add it to coverage or depend in define.yaml" % fullname.replace(".", "/") @@ -49,7 +60,7 @@ def cfg_parse(arg_list): """Parse the command line.""" parser = argparse.ArgumentParser(prog="python.py", description="Run a python test module.") - parser.add_argument("--lib", required=True, metavar="PATH", help="path of the harness library") + parser.add_argument("--lib", required=True, metavar="LIST", help="comma separated paths of the libraries the test may import") parser.add_argument("--test", required=True, metavar="PATH", help="path of the test module") parser.add_argument("--allow", default="", metavar="LIST", help="comma separated library modules the test may import") parser.add_argument("--name", default="", metavar="LIST", help="comma separated tests to run, or all of them when not set") @@ -59,7 +70,7 @@ def cfg_parse(arg_list): #################################################################################################################################### -def coverage_begin(path_lib): +def coverage_begin(path_lib_list): """Start measuring coverage, returning the measurement or None when it cannot be measured. Measuring starts before the test module is loaded, since a module imports the code it covers and the definitions in that code @@ -69,18 +80,18 @@ def coverage_begin(path_lib): import coverage except ImportError: print("unable to measure coverage: no module named 'coverage'") - print("HINT: install python3-coverage (d12, u24, rh10) or py3-coverage (a321, a324), else pip3 install 'coverage>=6.5'") + print("HINT: install python3-coverage (RHEL, Debian) or py3-coverage (Alpine), else pip3 install 'coverage>=6.5'") return None # Branch detail per line arrived in 6.5. An older one reports only summary counts, which cannot say which line to fix. if tuple(int(part) for part in coverage.__version__.split(".")[:2]) < (6, 5): print("coverage %s is too old, 6.5 or newer is required for branch detail" % coverage.__version__) - print("HINT: the version packaged for rh9 is too old, so pip3 install 'coverage>=6.5' there") + print("HINT: the version packaged is too old, so pip3 install 'coverage>=6.5' there") return None - result = coverage.Coverage(branch=True, data_file=None, source=[path_lib]) + result = coverage.Coverage(branch=True, data_file=None, source=path_lib_list) result.start() return result @@ -90,13 +101,19 @@ def coverage_begin(path_lib): def test_module_run(config): """Load the test module by path and run it with only the library modules it declared importable.""" - sys.path.insert(0, config.lib) - sys.meta_path.insert(0, ImportGuard(config.lib, filter(None, config.allow.split(",")))) + path_lib_list = config.lib.split(",") + + # The libraries the test may import from come first, in the order given, followed by the harness library the tests themselves + # need. Inserting in reverse leaves them on the path in that order. + for path_lib in reversed(path_lib_list + [PATH_HARNESS]): + sys.path.insert(0, path_lib) + + sys.meta_path.insert(0, ImportGuard(path_lib_list, filter(None, config.allow.split(",")))) measure = None if config.coverage is not None: - measure = coverage_begin(config.lib) + measure = coverage_begin(path_lib_list) if measure is None: return 2 diff --git a/test/lib/command/test/test.py b/test/lib/command/test/test.py index 768ee0dea..1bddf339a 100644 --- a/test/lib/command/test/test.py +++ b/test/lib/command/test/test.py @@ -15,7 +15,7 @@ from command.test.container import container_build, container_remove, container_ from command.test.define import test_def_parse from command.test.job import TestJob from command.test.list import test_list_get -from common.error import TestError, check +from common.error import ToolError, check from common.exec import Exec, exec_one from common.log import * from common.storage import file_read, file_write, file_write_differs, path_create @@ -90,29 +90,22 @@ def _repo_copy(config): #################################################################################################################################### def _code_generate(config, path_repo_copy): - """Generate the code that is built from the definitions in the source. + """Generate the code that is built from the declarations in the source. - This runs on the host rather than in a container since the generator is built for the host either way.""" + This runs on the host rather than in a container, since the generator is python and needs nothing built first.""" path_repo = config.repo_path - path_build = os.path.join(config.test_path, "build", VM_NONE) - build_clean = not os.path.exists(os.path.join(path_build, "build.ninja")) + command = "" - log(INFO, ("clean " if build_clean else "") + "autogenerate code") - - command = "ninja -C %s src/build-code 2>&1" % path_build + log(INFO, "autogenerate code") # A dry run does the minimum required, i.e. only what building the test list depends on if not config.dry_run: for generate in ("config", "error", "postgres-version"): - command += " && \\\n%s/src/build-code %s %s/src" % (path_build, generate, path_repo) + command += "%s/build/build.py %s && \\\n" % (path_repo, generate) - command += " && \\\n%s/src/build-code postgres %s/src %s" % (path_build, path_repo, path_repo_copy) - - if build_clean: - command = ( - "meson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug %s %s && \\\n" % (path_build, path_repo) + command - ) + # The PostgreSQL interfaces are generated into the repository copy, since they are built rather than committed + command += "%s/build/build.py postgres --build-path=%s" % (path_repo, path_repo_copy) exec_one(command) @@ -233,7 +226,7 @@ def _coverage(config, test_list): # The coverage command reports incomplete coverage with a status of one, which is not an error here since the tests all passed if status > 1: - raise TestError("coverage command failed", status) + raise ToolError("coverage command failed", status) if status == 0: log(INFO, "tested modules have full coverage") @@ -293,7 +286,12 @@ def cmd_test(config): for path in (config.test_path, os.path.join(path_repo, "test/result")): if os.path.exists(path): - exec_one("find %s -mindepth 1 -print0 | xargs -0 rm -rf" % path) + try: + exec_one("find %s -mindepth 1 -print0 | xargs -0 rm -rf" % path) + except ToolError as error: + # A test that did not get to clean up after itself can leave files owned by root behind, which is the only thing + # here that cannot be removed as the user the tests run as + raise ToolError("%s\nHINT: a test may have left files owned by root, so try 'sudo rm -rf %s/*'" % (error, path)) if config.clean_only: return 0 @@ -398,7 +396,7 @@ def cmd_test(config): return 0 if not test_list: - raise TestError("no tests were selected") + raise ToolError("no tests were selected") log(INFO, "%u test%s selected\n" % (len(test_list), "" if len(test_list) == 1 else "s")) diff --git a/test/lib/command/test/unit.py b/test/lib/command/test/unit.py index e5a0d078e..1e5d32206 100644 --- a/test/lib/command/test/unit.py +++ b/test/lib/command/test/unit.py @@ -13,8 +13,16 @@ import sys from command.lint.lint import cmd_lint from command.test.build import TestBuild -from command.test.define import TEST_LANG_PYTHON, TEST_TYPE_PERFORMANCE, test_def_find, test_def_parse -from common.error import TestError, check +from command.test.define import ( + TEST_LANG_PYTHON, + TEST_LIB_LIST, + TEST_TYPE_PERFORMANCE, + test_def_find, + test_def_parse, + test_lib_path, + test_lib_split, +) +from common.error import ToolError, check from common.exec import exec_one from common.log import * from common.storage import file_remove, path_create, path_list @@ -41,9 +49,7 @@ def _path_recreate(path): def _python_module(name): """Convert a code module name to the module it is imported as, e.g. test/common/string_id becomes common.string_id.""" - check(name.startswith("test/"), "python module '%s' must be under test/" % name) - - return name[len("test/") :].replace("/", ".") + return test_lib_split(name)[1].replace("/", ".") #################################################################################################################################### @@ -59,19 +65,22 @@ def _test_python(config, module): path_test = os.path.join(config.repo_path, "test/src/module", module.name + "_test.py") if not os.path.isfile(path_test): - raise TestError("unable to find test module '%s'" % path_test) + raise ToolError("unable to find test module '%s'" % path_test) # What the test may import: the code modules it covers, plus what it declared, plus what earlier tests covered allow_list = [_python_module(coverage.name) for coverage in module.coverage_list if coverage.coverable] allow_list += [_python_module(name) for name in list(module.depend_list) + list(module.include_list)] + # Libraries the test may import from, which is the one the test module lives in and the ones below it in the hierarchy. A test + # of the build library cannot reach the test library this way, the same as the build tool itself cannot. + path_lib_list = [os.path.join(config.repo_path, test_lib_path(lib)) for lib in TEST_LIB_LIST[test_lib_split(module.name)[0]]] + # A single test can be run while debugging with --test-name, e.g. # test.py unit test/common/string --test-name=test_string_id_render - path_lib = os.path.join(config.repo_path, "test/lib") command = "%s '%s/command/test/python.py' --lib='%s' --test='%s' --allow='%s' --name='%s'" % ( sys.executable, - path_lib, - path_lib, + os.path.join(config.repo_path, test_lib_path("test")), + ",".join(path_lib_list), path_test, ",".join(allow_list), config.test_name or "", @@ -115,7 +124,7 @@ def cmd_unit(config): if module.flag is not None or config.profile or module.type == TEST_TYPE_PERFORMANCE: if module.flag is not None and module.flag != "-DNDEBUG": - raise TestError("unexpected define '%s'" % module.flag) + raise ToolError("unexpected define '%s'" % module.flag) meson_setup += "release" else: @@ -153,10 +162,10 @@ def cmd_unit(config): exec_one("ninja -C '%s'" % path_unit_build) break - except TestError as error: + except ToolError as error: # If this is the first build failure then clean the build path and retry if build_retry: - raise TestError("build failed for unit %s: %s" % (config.module, error)) + raise ToolError("build failed for unit %s: %s" % (config.module, error)) build_retry = True diff --git a/test/lib/common/exec.py b/test/lib/common/exec.py index b8c3ce3c1..e7c1cfbb6 100644 --- a/test/lib/common/exec.py +++ b/test/lib/common/exec.py @@ -3,7 +3,7 @@ Exec runs a command in the background so several can run at once, which is how the test driver keeps every vm busy. It keeps stdout and stderr apart because a test that writes anything to stderr has failed whatever its exit status, e.g. a valgrind error. -exec_one() runs a command and waits for it, with stderr folded into the output as execOneExpectP() in src/build/common/exec.c does. +exec_one() runs a command and waits for it, with stderr folded into the output as execOneExpectP() in test/src/harness/exec.c does. The combined output is returned on success and carried in the error on failure, since a failed meson or ninja run says what went wrong there rather than in its exit status.""" @@ -13,7 +13,7 @@ import select import subprocess import sys -from common.error import TestError +from common.error import ToolError #################################################################################################################################### @@ -108,7 +108,7 @@ class Exec: def exec_one(command, result_expect=0, show_output=False): """Run a command through the shell and return its combined output. - Raise TestError when the exit status is not the expected one.""" + Raise ToolError when the exit status is not the expected one.""" process = Exec(command, show_output=show_output, merge_error=True) process.begin() @@ -117,7 +117,7 @@ def exec_one(command, result_expect=0, show_output=False): if status != result_expect: output = process.output.strip() - raise TestError("%s terminated unexpectedly [%d]%s%s" % (command, status, ": " if output else "", output)) + raise ToolError("%s terminated unexpectedly [%d]%s%s" % (command, status, ": " if output else "", output)) return process.output diff --git a/test/lib/common/render.py b/test/lib/common/render.py deleted file mode 100644 index a397e7ea4..000000000 --- a/test/lib/common/render.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Render Helpers. - -Port of the naming helpers in src/build/common/render.c.""" - -#################################################################################################################################### -from common.string_id import STRING_ID_SEQ_NONE, str_id_from_z, str_id_seq - - -#################################################################################################################################### -def bld_enum(prefix, value): - """Convert a dashed value to an enum name, e.g. "type-convert" becomes "typeConvert". - - When a prefix is given it is prepended and the first letter of the value is upper-cased, e.g. bld_enum("logLevel", "info") - becomes "logLevelInfo".""" - - result = "" - upper = False - - if prefix is not None: - result = prefix - upper = True - - for char in value: - # A dash is removed and the letter that follows it is upper-cased - if char == "-": - upper = True - continue - - result += char.upper() if upper else char - upper = False - - return result - - -#################################################################################################################################### -def bld_str_id_seq(value, sequence=STRING_ID_SEQ_NONE): - """Render the StringId macro for a value, e.g. STRID5("test", 0x2a7250), which is the form the source is expected to contain.""" - - str_id = str_id_from_z(value, sequence) - - if sequence is STRING_ID_SEQ_NONE: - return 'STRID%u("%s", 0x%x)' % ((str_id & 1) + 5, value, str_id) - - return 'STRID%uS("%s", %u, 0x%x)' % ((str_id & 1) + 5, value, str_id_seq(str_id), str_id) diff --git a/test/lib/common/vm.py b/test/lib/common/vm.py index e5fb7db6c..fc61e6bd9 100644 --- a/test/lib/common/vm.py +++ b/test/lib/common/vm.py @@ -9,7 +9,7 @@ vm and at least one default vm must collect C coverage. Without the check a new #################################################################################################################################### import platform -from common.error import TestError, check +from common.error import ToolError, check # Os base a vm is built on, which determines its package manager and where PostgreSQL is installed VM_OS_BASE_ALPINE = "alpine" @@ -113,7 +113,7 @@ def vm_get(vm): """The definition for a vm, which also validates that the vm exists.""" if not vm_valid(vm): - raise TestError("no definition for vm '%s'" % vm) + raise ToolError("no definition for vm '%s'" % vm) return _VM[vm] diff --git a/test/lib/config/config.py b/test/lib/config/config.py index dac27970f..f7f70a240 100644 --- a/test/lib/config/config.py +++ b/test/lib/config/config.py @@ -5,7 +5,7 @@ Parses the command line, resolves paths, and initializes logging.""" import os import re -from common.error import TestError +from common.error import ToolError from common.log import * from common.storage import file_read from config.cli import cli_parse @@ -23,7 +23,7 @@ def project_version(path_repo): match = re.search(r'^#define PROJECT_VERSION\s+"([^"]+)"', file_read(os.path.join(path_repo, "src/version.h")), re.M) if match is None: - raise TestError("unable to find PROJECT_VERSION in src/version.h") + raise ToolError("unable to find PROJECT_VERSION in src/version.h") return match.group(1) @@ -42,7 +42,7 @@ def cfg_load(arg_list, path_harness): # The repository is only ever read from so it must already exist. Checking here keeps a mistyped path from looking like success # rather than an error, e.g. the linter would find no files to check and report that everything passed. if not os.path.isdir(config.repo_path): - raise TestError("repo path '%s' does not exist" % config.repo_path) + raise ToolError("repo path '%s' does not exist" % config.repo_path) # Convert log levels to ids now that they are known to be valid config.log_level = OFF if config.quiet else log_level_parse(config.log_level) diff --git a/test/lib/harness/test.py b/test/lib/harness/test.py index e1470a4ba..417d26024 100644 --- a/test/lib/harness/test.py +++ b/test/lib/harness/test.py @@ -31,7 +31,7 @@ assert_is_instance = _case.assertIsInstance assert_in = _case.assertIn assert_not_in = _case.assertNotIn -# Errors, used as a context manager, e.g. with assert_raises(TestError) as error: +# Errors, used as a context manager, e.g. with assert_raises(ToolError) as error: assert_raises = _case.assertRaises # What a test module gets from "from harness.test import *". Listing it here means a test module never has to update its import diff --git a/test/smoke.py b/test/smoke.py index e12f3f224..e7dd59ade 100755 --- a/test/smoke.py +++ b/test/smoke.py @@ -258,9 +258,8 @@ def version_parse(version): def supported_versions(): """Read the PostgreSQL versions supported by pgBackRest from the generated version header. - The header is generated from src/build/postgres/postgres.yaml and, unlike that file, is shipped in the distribution, so this - works from a source checkout or a distribution tarball. Return None when the header cannot be found so that every version found - is tested. + The header is generated from build/postgres.yaml and, unlike that file, is shipped in the distribution, so this works from a + source checkout or a distribution tarball. Return None when the header cannot be found so that every version found is tested. """ path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "src", "postgres", "version.auto.h") diff --git a/test/src/harness/config.c b/test/src/harness/config.c index 48ba45322..a28bed19d 100644 --- a/test/src/harness/config.c +++ b/test/src/harness/config.c @@ -7,11 +7,11 @@ Harness for Loading Test Configurations #include #include -#include "build/common/string.h" #include "common/io/io.h" #include "config/config.intern.h" #include "config/load.h" #include "config/parse.h" +#include "harness/string.h" #include "storage/helper.h" #include "version.h" @@ -223,7 +223,8 @@ FN_INLINE_ALWAYS const char * hrnCfgEnvName(const ConfigOption optionId, const unsigned optionKey) { return strZ( - strReplaceChr(strUpper(strNewFmt(HRN_PGBACKREST_ENV "%s", cfgParseOptionKeyIdxName(optionId, optionKey - 1))), '-', '_')); + strReplaceChr( + hrnStrUpper(strNewFmt(HRN_PGBACKREST_ENV "%s", cfgParseOptionKeyIdxName(optionId, optionKey - 1))), '-', '_')); } void diff --git a/src/build/common/exec.c b/test/src/harness/exec.c similarity index 94% rename from src/build/common/exec.c rename to test/src/harness/exec.c index 36be5c32d..1e4934732 100644 --- a/src/build/common/exec.c +++ b/test/src/harness/exec.c @@ -4,12 +4,12 @@ Execute Process Extensions // Include core module #include "common/exec.c" -#include "build/common/exec.h" #include "common/io/fd.h" +#include "harness/exec.h" /**********************************************************************************************************************************/ static String * -execProcess(Exec *const this, const ExecOneExpectParam param) +execProcess(Exec *const this, const HrnExecOneExpectParam param) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(EXEC, this); @@ -50,7 +50,7 @@ execProcess(Exec *const this, const ExecOneExpectParam param) /**********************************************************************************************************************************/ FN_EXTERN String * -execOneExpect(const String *const command, const ExecOneExpectParam param) +hrnExecOneExpect(const String *const command, const HrnExecOneExpectParam param) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING, command); diff --git a/src/build/common/exec.h b/test/src/harness/exec.h similarity index 79% rename from src/build/common/exec.h rename to test/src/harness/exec.h index a241ae964..a183add05 100644 --- a/src/build/common/exec.h +++ b/test/src/harness/exec.h @@ -4,8 +4,8 @@ Execute Process Extensions Functions intended to simplify exec'ing and getting output. The core Exec object is efficient but it does not work well for the requirements of the build, test, and doc which prefer ease of use. ***********************************************************************************************************************************/ -#ifndef BUILD_COMMON_EXEC_H -#define BUILD_COMMON_EXEC_H +#ifndef TEST_HARNESS_EXEC_H +#define TEST_HARNESS_EXEC_H #include "common/exec.h" @@ -13,17 +13,17 @@ requirements of the build, test, and doc which prefer ease of use. Functions ***********************************************************************************************************************************/ // Execute a command similar to system() while also capturing output. Note that stderr is redirected to stdout. -typedef struct ExecOneExpectParam +typedef struct HrnExecOneExpectParam { VAR_PARAM_HEADER; const String *shell; // Shell command to use for exec (default is sh -c) int resultExpect; // Expected result, if not 0 TimeMSec timeout; // Command timeout (default is ioTimeoutMs()) -} ExecOneExpectParam; +} HrnExecOneExpectParam; -#define execOneExpectP(command, ...) \ - execOneExpect(command, (ExecOneExpectParam){VAR_PARAM_INIT, __VA_ARGS__}) +#define hrnExecOneExpectP(command, ...) \ + hrnExecOneExpect(command, (HrnExecOneExpectParam){VAR_PARAM_INIT, __VA_ARGS__}) -FN_EXTERN String *execOneExpect(const String *command, ExecOneExpectParam param); +FN_EXTERN String *hrnExecOneExpect(const String *command, HrnExecOneExpectParam param); #endif diff --git a/test/src/harness/host.c b/test/src/harness/host.c index 235a69002..dc4364e79 100644 --- a/test/src/harness/host.c +++ b/test/src/harness/host.c @@ -3,7 +3,6 @@ Host Harness ***********************************************************************************************************************************/ #include -#include "build/common/exec.h" #include "common/compress/helper.h" #include "common/crypto/common.h" #include "common/error/retry.h" @@ -11,6 +10,7 @@ Host Harness #include "common/type/json.h" #include "common/wait.h" #include "config/config.h" +#include "harness/exec.h" #include "postgres/interface.h" #include "postgres/version.h" #include "storage/azure/storage.intern.h" @@ -157,7 +157,7 @@ hrnHostNew(const StringId id, const String *const container, const String *const lstAdd(hrnHostLocal.hostList, &this); // Remove prior container with same name if it exists - execOneExpectP(strNewFmt("docker rm -f %s", strZ(hrnHostContainer(this)))); + hrnExecOneExpectP(strNewFmt("docker rm -f %s", strZ(hrnHostContainer(this)))); // Run container String *const command = strCatFmt( @@ -184,11 +184,11 @@ hrnHostNew(const StringId id, const String *const container, const String *const strCatFmt(command, " %s", strZ(param.param)); // Allow extra time since the image may need to be pulled on first use (some object store images are large) - execOneExpectP(command, .timeout = 600000); + hrnExecOneExpectP(command, .timeout = 600000); // Get IP address const String *const ip = strTrim( - execOneExpectP( + hrnExecOneExpectP( strNewFmt( "docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' %s", strZ(hrnHostContainer(this))))); @@ -254,7 +254,7 @@ hrnHostExec(HrnHost *const this, const String *const command, const HrnHostExecP { strCat( result, - execOneExpectP( + hrnExecOneExpectP( command, .shell = strNewFmt( "docker exec -u %s %s sh -c", param.user == NULL ? strZ(hrnHostUser(this)) : strZ(param.user), diff --git a/test/src/harness/log.c b/test/src/harness/log.c index 1936205b4..186b1d18d 100644 --- a/test/src/harness/log.c +++ b/test/src/harness/log.c @@ -9,10 +9,10 @@ Log Test Harness #include #include -#include "build/common/regExp.h" #include "common/log.h" #include "common/memContext.h" #include "common/type/stringList.h" +#include "harness/regExp.h" #include "harness/debug.h" #include "harness/log.h" @@ -364,11 +364,11 @@ hrnLogReplace(void) while (regExpMatch(logReplace->regExp, STRDEF(harnessLogBuffer))) { // Get the match - String *match = regExpMatchStr(logReplace->regExp, STRDEF(harnessLogBuffer)); + String *match = hrnRegExpMatchStr(logReplace->regExp, STRDEF(harnessLogBuffer)); // Find beginning of match char *begin = - harnessLogBuffer + (regExpMatchPtr(logReplace->regExp, STRDEF(harnessLogBuffer)) - harnessLogBuffer); + harnessLogBuffer + (hrnRegExpMatchPtr(logReplace->regExp, STRDEF(harnessLogBuffer)) - harnessLogBuffer); // If there is a sub expression then evaluate it if (logReplace->regExpSub != NULL) @@ -382,10 +382,10 @@ hrnLogReplace(void) } // Find beginning of match - begin += regExpMatchPtr(logReplace->regExpSub, match) - strZ(match); + begin += hrnRegExpMatchPtr(logReplace->regExpSub, match) - strZ(match); // Get the match - match = regExpMatchStr(logReplace->regExpSub, match); + match = hrnRegExpMatchStr(logReplace->regExpSub, match); } // Build replacement string. If versioned then append the version number. diff --git a/src/build/common/regExp.c b/test/src/harness/regExp.c similarity index 92% rename from src/build/common/regExp.c rename to test/src/harness/regExp.c index 132d7d2c5..d1b938fc8 100644 --- a/src/build/common/regExp.c +++ b/test/src/harness/regExp.c @@ -4,13 +4,13 @@ Regular Expression Handler Extensions // Include core module #include "common/regExp.c" -#include "build/common/regExp.h" +#include "harness/regExp.h" /*********************************************************************************************************************************** Getters/Setters ***********************************************************************************************************************************/ const char * -regExpMatchPtr(RegExp *const this, const String *const string) +hrnRegExpMatchPtr(RegExp *const this, const String *const string) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(REGEXP, this); @@ -36,7 +36,7 @@ regExpMatchPtr(RegExp *const this, const String *const string) } String * -regExpMatchStr(RegExp *const this, const String *const string) +hrnRegExpMatchStr(RegExp *const this, const String *const string) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(REGEXP, this); diff --git a/src/build/common/regExp.h b/test/src/harness/regExp.h similarity index 79% rename from src/build/common/regExp.h rename to test/src/harness/regExp.h index 463dd7370..d64e78a40 100644 --- a/src/build/common/regExp.h +++ b/test/src/harness/regExp.h @@ -1,8 +1,8 @@ /*********************************************************************************************************************************** Regular Expression Handler Extensions ***********************************************************************************************************************************/ -#ifndef BUILD_COMMON_REGEXP_H -#define BUILD_COMMON_REGEXP_H +#ifndef TEST_HARNESS_REGEXP_H +#define TEST_HARNESS_REGEXP_H #include "common/regExp.h" @@ -10,9 +10,9 @@ Regular Expression Handler Extensions Getters/Setters ***********************************************************************************************************************************/ // Get pointer to the last match. NULL if there was no match. -const char *regExpMatchPtr(RegExp *this, const String *string); +const char *hrnRegExpMatchPtr(RegExp *this, const String *string); // Get the last match as a String. NULL if there was no match. -String *regExpMatchStr(RegExp *this, const String *string); +String *hrnRegExpMatchStr(RegExp *this, const String *string); #endif diff --git a/src/build/common/string.c b/test/src/harness/string.c similarity index 93% rename from src/build/common/string.c rename to test/src/harness/string.c index 3270d5f57..aac254ede 100644 --- a/src/build/common/string.c +++ b/test/src/harness/string.c @@ -4,11 +4,11 @@ String Handler Extensions // Include core module #include "common/type/string.c" -#include "build/common/string.h" +#include "harness/string.h" /**********************************************************************************************************************************/ String * -strUpper(String *const this) +hrnStrUpper(String *const this) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(STRING, this); @@ -24,7 +24,7 @@ strUpper(String *const this) /**********************************************************************************************************************************/ unsigned int -strReplace(String *const this, const String *const replace, const String *const with) +hrnStrReplace(String *const this, const String *const replace, const String *const with) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(STRING, this); diff --git a/src/build/common/string.h b/test/src/harness/string.h similarity index 80% rename from src/build/common/string.h rename to test/src/harness/string.h index 8e7abaede..fc28a0de4 100644 --- a/src/build/common/string.h +++ b/test/src/harness/string.h @@ -1,8 +1,8 @@ /*********************************************************************************************************************************** String Handler Extensions ***********************************************************************************************************************************/ -#ifndef BUILD_COMMON_STRING_H -#define BUILD_COMMON_STRING_H +#ifndef TEST_HARNESS_STRING_H +#define TEST_HARNESS_STRING_H #include "common/type/string.h" @@ -10,9 +10,9 @@ String Handler Extensions Functions ***********************************************************************************************************************************/ // Upper-case entire string -String *strUpper(String *this); +String *hrnStrUpper(String *this); // Replace a substring with another string. Returns the number of replacements made. -unsigned int strReplace(String *this, const String *replace, const String *with); +unsigned int hrnStrReplace(String *this, const String *replace, const String *with); #endif diff --git a/test/src/module/build/command/main_test.py b/test/src/module/build/command/main_test.py new file mode 100644 index 000000000..316a45aca --- /dev/null +++ b/test/src/module/build/command/main_test.py @@ -0,0 +1,185 @@ +"""Test Build Command. + +Each generator has its own test, so what is checked here is that the command line reaches the right one and that it writes where it was +told to.""" + +#################################################################################################################################### +import io +import os +import tempfile +from contextlib import redirect_stderr + +from harness.test import * + +from common.error import * +from common.storage import file_write +from command.main import * + +# The smallest declarations each generator will accept, since what each of them renders is checked with the generator itself +ERROR = "assert:\n code: 25\n fatal: true\n" + +CONFIG = """command: + backup: {} + +optionGroup: + pg: {} + +option: + buffer-size: + type: size + default: 1MiB + + io-timeout: + type: time + default: 60 + + process-max: + type: integer + default: 1 + + stanza: + type: string +""" + +HELP = """ + + + + + + + + + + + + + + + + + + + + Backup. + + +

Backup.

+ + + + + +""" + +POSTGRES = "version:\n - 10\n" +VENDOR = "typedef uint32 TransactionId;\n" +INTERN = "#define PG_INTERFACE_CONTROL_IS(version)\n" + + +#################################################################################################################################### +def _repo_create(path): + """Write every declaration the generators read.""" + + file_write(os.path.join(path, "build/error.yaml"), ERROR) + file_write(os.path.join(path, "build/config.yaml"), CONFIG) + file_write(os.path.join(path, "build/postgres.yaml"), POSTGRES) + file_write(os.path.join(path, "doc/xml/reference.xml"), HELP) + file_write(os.path.join(path, "src/postgres/interface/version.vendor.h"), VENDOR) + file_write(os.path.join(path, "src/postgres/interface/version.intern.h"), INTERN) + + +#################################################################################################################################### +def test_cfg_load(): + """The repository is the one the tool is part of unless the caller names another, and the generated code goes there too.""" + + config = cfg_load(["error"], "/repo") + + assert_equal(config.command, "error") + assert_equal(config.repo_path, "/repo") + assert_equal(config.build_path, "/repo") + + # The documentation generates from its own declarations, so it names the repository to read them from + config = cfg_load(["config", "--repo-path=/repo/doc"], "/repo") + + assert_equal(config.repo_path, "/repo/doc") + assert_equal(config.build_path, "/repo/doc") + + # Meson sends the code it generates as it builds to the build directory rather than to the repository + config = cfg_load(["help", "--repo-path=/repo", "--build-path=/build"], "/repo") + + assert_equal(config.repo_path, "/repo") + assert_equal(config.build_path, "/build") + + +#################################################################################################################################### +def test_cfg_load_error(): + """A command that has no generator is refused with the list of the ones that do.""" + + output = io.StringIO() + + with assert_raises(SystemExit): + with redirect_stderr(output): + cfg_load(["bogus"], "/repo") + + assert_in("invalid choice: 'bogus'", output.getvalue()) + + +#################################################################################################################################### +def test_cmd_build(): + """The command line reaches the generator it names, which writes where it was told to.""" + + generated = { + "config": ("src/config/config.auto.h", "src/config/parse.auto.c.inc"), + "error": ("src/common/error/error.auto.h", "src/common/error/error.auto.c.inc"), + "help": ("src/command/help/help.auto.c.inc",), + "help-data": ("help.dat",), + "postgres": ("src/postgres/interface.auto.c.inc",), + "postgres-version": ("src/postgres/version.auto.h",), + } + + with tempfile.TemporaryDirectory() as path: + _repo_create(path) + + for command, file_list in generated.items(): + path_build = os.path.join(path, "out", command) + + cmd_build(cfg_load([command, "--repo-path=%s" % path, "--build-path=%s" % path_build], path)) + + for file in file_list: + assert_true(os.path.isfile(os.path.join(path_build, file)), "%s did not generate %s" % (command, file)) + + +#################################################################################################################################### +def test_cmd_build_missing(): + """A declaration that is not there is reported with the path it was looked for at.""" + + with tempfile.TemporaryDirectory() as path: + with assert_raises(ToolError) as error: + cmd_build(cfg_load(["error", "--repo-path=%s" % path], path)) + + assert_equal(str(error.exception), "unable to open file '%s/build/error.yaml' for read" % path) diff --git a/test/src/module/test/common/error_test.py b/test/src/module/build/common/error_test.py similarity index 90% rename from test/src/module/test/common/error_test.py rename to test/src/module/build/common/error_test.py index 672ed9aa9..77172a208 100644 --- a/test/src/module/test/common/error_test.py +++ b/test/src/module/build/common/error_test.py @@ -10,7 +10,7 @@ from common.error import * def test_error_message(): """The message is what gets reported and the status is what the harness exits with.""" - error = TestError("something went wrong") + error = ToolError("something went wrong") assert_equal(str(error), "something went wrong") assert_equal(error.status, EXIT_ERROR) @@ -25,8 +25,8 @@ def test_error_status(): That is how the coverage command reports incomplete coverage rather than a failure.""" - assert_equal(TestError("incomplete", 1).status, 1) - assert_equal(TestError("failed", 0).status, 0) + assert_equal(ToolError("incomplete", 1).status, 1) + assert_equal(ToolError("failed", 0).status, 0) # The default is distinct from the incomplete coverage status so the two are never confused assert_not_equal(EXIT_ERROR, 1) @@ -45,7 +45,7 @@ def test_check_fail(): """A condition that does not hold raises with the message given.""" for condition in (False, 0, "", [], {}, None): - with assert_raises(TestError) as error: + with assert_raises(ToolError) as error: check(condition, "invariant does not hold") assert_equal(str(error.exception), "invariant does not hold") diff --git a/test/src/module/test/common/log_test.py b/test/src/module/build/common/log_test.py similarity index 100% rename from test/src/module/test/common/log_test.py rename to test/src/module/build/common/log_test.py diff --git a/test/src/module/build/common/pack_test.py b/test/src/module/build/common/pack_test.py new file mode 100644 index 000000000..7466c3d2e --- /dev/null +++ b/test/src/module/build/common/pack_test.py @@ -0,0 +1,151 @@ +"""Test Pack Writer. + +The bytes expected here are read the way the format documents them, i.e. a tag byte holding the type, the value, and the gap since the +last field, so a failure says which part of the tag is wrong rather than only that a byte differs.""" + +#################################################################################################################################### +from harness.test import * + +from common.error import * +from common.pack import * + + +#################################################################################################################################### +def test_pack_bool(): + """A boolean is one bit of the tag, and false is left null since that is the default.""" + + pack = PackWrite() + pack.bool_write(True) + + # 2 = bool type, 8 = value bit + assert_equal(pack.end(), b"\x28\x00") + + # False writes nothing at all, so the pack holds only its terminator + pack = PackWrite() + pack.bool_write(False) + + assert_equal(pack.end(), b"\x00") + + # Unless writing it is forced, which is what says the value is there and is false + pack = PackWrite() + pack.bool_write(False, default_write=True) + + assert_equal(pack.end(), b"\x20\x00") + + +#################################################################################################################################### +def test_pack_str(): + """A string is a tag saying whether there is data, then the size and the bytes.""" + + pack = PackWrite() + pack.str_write("sample") + + # 7 = string type, 8 = data bit, 06 = size, then the bytes + assert_equal(pack.end(), b"\x78\x06sample\x00") + + # An empty string is written, since the tag says whether there is a string rather than how long it is + pack = PackWrite() + pack.str_write("") + + assert_equal(pack.end(), b"\x70\x00") + + # No string at all is left null + pack = PackWrite() + pack.str_write(None) + + assert_equal(pack.end(), b"\x00") + + # A string longer than a single byte size, which is where the size takes two bytes + pack = PackWrite() + pack.str_write("x" * 200) + + assert_equal(pack.end(), b"\x78\xc8\x01" + b"x" * 200 + b"\x00") + + +#################################################################################################################################### +def test_pack_id(): + """A field that is left null leaves a gap in the ids, which is what says it was there.""" + + # A gap of one is never recorded, since there is always at least that much, so one null leaves a recorded gap of one + pack = PackWrite() + pack.null_write() + pack.bool_write(True) + + # 2 = bool, 8 = value bit, 1 = the low bits of the gap + assert_equal(pack.end(), b"\x29\x00") + + # A gap too wide for the two bits the tag has for it, which is what the more-gap bit says + pack = PackWrite() + + for _ in range(40): + pack.null_write() + + pack.bool_write(True) + + # 4 = more gap follows, then the gap shifted right by the two bits already in the tag + assert_equal(pack.end(), b"\x2c\x0a\x00") + + # An id given explicitly leaves the same gap as the nulls it skips over would + pack = PackWrite() + pack.bool_write(True, id=4) + + assert_equal(pack.end(), b"\x2b\x00") + + pack = PackWrite() + pack.bool_write(True, id=2) + + with assert_raises(ToolError) as error: + pack.bool_write(True, id=1) + + assert_equal(str(error.exception), "field id must be greater than last id") + + +#################################################################################################################################### +def test_pack_container(): + """A container numbers its fields from one and is ended with a zero.""" + + pack = PackWrite() + pack.array_begin() + pack.bool_write(True) + pack.bool_write(True) + pack.array_end() + + # 1 = array type, then two bools, then the array end and the pack end + assert_equal(pack.end(), b"\x10\x28\x28\x00\x00") + + # An object at an id of its own, which is how the reader knows which command an override is for + pack = PackWrite() + pack.obj_begin(id=2) + pack.str_write("x") + pack.obj_end() + + assert_equal(pack.end(), b"\x51\x78\x01x\x00\x00") + + # A container has three bits of the tag for the gap rather than two, since it has no value to hold + pack = PackWrite() + pack.array_begin(id=9) + pack.array_end() + + assert_equal(pack.end(), b"\x18\x01\x00\x00") + + # A container must be ended before the pack is, and with the end that matches it + pack = PackWrite() + pack.array_begin() + + with assert_raises(ToolError) as error: + pack.obj_end() + + assert_equal(str(error.exception), "not in object") + + with assert_raises(ToolError) as error: + pack.end() + + assert_equal(str(error.exception), "container not ended") + + pack = PackWrite() + pack.obj_begin() + + with assert_raises(ToolError) as error: + pack.array_end() + + assert_equal(str(error.exception), "not in array") diff --git a/test/src/module/test/common/render_test.py b/test/src/module/build/common/render_test.py similarity index 100% rename from test/src/module/test/common/render_test.py rename to test/src/module/build/common/render_test.py diff --git a/test/src/module/test/common/storage_test.py b/test/src/module/build/common/storage_test.py similarity index 91% rename from test/src/module/test/common/storage_test.py rename to test/src/module/build/common/storage_test.py index c25d5cf4f..2322ce366 100644 --- a/test/src/module/test/common/storage_test.py +++ b/test/src/module/build/common/storage_test.py @@ -30,6 +30,12 @@ def test_file_read_write(): assert_equal(file_read(path_file), "replaced") + # Bytes are written as bytes, which is how the raw help data is written for the C test to load + file_write(path_file, b"\x00\x01raw") + + with open(path_file, "rb") as file: + assert_equal(file.read(), b"\x00\x01raw") + #################################################################################################################################### def test_file_read_error(): @@ -38,7 +44,7 @@ def test_file_read_error(): with tempfile.TemporaryDirectory() as path: path_file = os.path.join(path, "missing.txt") - with assert_raises(TestError) as error: + with assert_raises(ToolError) as error: file_read(path_file) assert_equal(str(error.exception), "unable to open file '%s' for read" % path_file) @@ -47,7 +53,7 @@ def test_file_read_error(): assert_is_none(file_read(path_file, ignore_missing=True)) # Anything else reports what the system said, and is an error however the caller asked for it - with assert_raises(TestError) as error: + with assert_raises(ToolError) as error: file_read(path, ignore_missing=True) assert_equal(str(error.exception), "unable to open file '%s' for read: Is a directory" % path) @@ -58,7 +64,7 @@ def test_file_write_error(): """A file that cannot be written reports what the system said.""" with tempfile.TemporaryDirectory() as path: - with assert_raises(TestError) as error: + with assert_raises(ToolError) as error: file_write(path, "content") assert_equal(str(error.exception), "unable to open file '%s' for write: Is a directory" % path) @@ -104,13 +110,13 @@ def test_file_remove(): # Missing is not an error by default since a file that is gone is what the caller wanted assert_is_none(file_remove(path_file)) - with assert_raises(TestError) as error: + with assert_raises(ToolError) as error: file_remove(path_file, error_on_missing=True) assert_equal(str(error.exception), "unable to remove missing file '%s'" % path_file) # Anything else reports what the system said - with assert_raises(TestError) as error: + with assert_raises(ToolError) as error: file_remove(path) assert_equal(str(error.exception), "unable to remove file '%s': Is a directory" % path) @@ -135,7 +141,7 @@ def test_path_create(): path_file = os.path.join(path, "file.txt") file_write(path_file, "content") - with assert_raises(TestError) as error: + with assert_raises(ToolError) as error: path_create(os.path.join(path_file, "sub")) assert_equal(str(error.exception), "unable to create path '%s': Not a directory" % os.path.join(path_file, "sub")) @@ -160,7 +166,7 @@ def test_path_list(): assert_equal(path_list(path_missing), []) - with assert_raises(TestError) as error: + with assert_raises(ToolError) as error: path_list(path_missing, error_on_missing=True) assert_equal(str(error.exception), "unable to list missing path '%s'" % path_missing) @@ -168,7 +174,7 @@ def test_path_list(): # Anything else reports what the system said path_file = os.path.join(path, "a.txt") - with assert_raises(TestError) as error: + with assert_raises(ToolError) as error: path_list(path_file) assert_equal(str(error.exception), "unable to list path '%s': Not a directory" % path_file) diff --git a/test/src/module/test/common/string_test.py b/test/src/module/build/common/string_test.py similarity index 95% rename from test/src/module/test/common/string_test.py rename to test/src/module/build/common/string_test.py index a93e7fea6..bb991cc58 100644 --- a/test/src/module/test/common/string_test.py +++ b/test/src/module/build/common/string_test.py @@ -69,7 +69,7 @@ def test_string_id_encode_error(): # Characters outside both encodings for value in ("under_score", "dot.dot", "spa ce", "plus+"): - with assert_raises(TestError) as error: + with assert_raises(ToolError) as error: str_id_from_z(value) assert_equal(str(error.exception), "'%s' contains invalid characters" % value) @@ -77,17 +77,17 @@ def test_string_id_encode_error(): # Longer than 5-bit holds, and 6-bit holds fewer characters still so it cannot take over assert_not_equal(str_id_from_z("a" * STRID5_MAX), 0) - with assert_raises(TestError): + with assert_raises(ToolError): str_id_from_z("a" * (STRID5_MAX + 1)) # Upper case forces 6-bit, which holds fewer characters assert_not_equal(str_id_from_z("A" * STRID6_MAX), 0) - with assert_raises(TestError): + with assert_raises(ToolError): str_id_from_z("A" * (STRID6_MAX + 1)) # An empty string has nothing to encode - with assert_raises(TestError) as error: + with assert_raises(ToolError) as error: str_id_from_z("") assert_equal(str(error.exception), "StringId value may not be empty") diff --git a/test/src/module/build/common/xml_test.py b/test/src/module/build/common/xml_test.py new file mode 100644 index 000000000..e2ee66d7d --- /dev/null +++ b/test/src/module/build/common/xml_test.py @@ -0,0 +1,232 @@ +"""Test Xml Handler. + +Mixed content is what most of this is about, so the checks are written as the xml that comes out rather than as the tree, since that is +where an error in where text ended up actually shows.""" + +#################################################################################################################################### +import xml.etree.ElementTree as etree + +from harness.test import * + +from common.error import * +from common.xml import * + + +#################################################################################################################################### +def _render(node): + """Render a node as the xml text of it.""" + + return etree.tostring(node, encoding="unicode") + + +#################################################################################################################################### +def test_xml_document(): + """A document is a root node and the declarations above it.""" + + document = xml_document_new("doc") + xml_node_attribute_set(document.root, "title", "Reference") + + assert_equal(document.render(), '\n\n') + + # A document type, which is what says where the tags it uses are declared + document = xml_document_new("doc", dtd_name="doc", dtd_file="doc.dtd") + xml_node_add(document.root, "p") + + assert_equal( + document.render(), + '\n\n

\n', + ) + + +#################################################################################################################################### +def test_xml_document_parse(): + """A document that is read and written again declares the same document type.""" + + document = xml_document_parse('\n\n

text

\n', "test.xml") + + assert_equal(document.dtd_name, "doc") + assert_equal(document.dtd_file, "doc.dtd") + assert_equal( + document.render(), '\n\n

text

\n' + ) + + # A document that declares none + document = xml_document_parse("

text

", "test.xml") + + assert_is_none(document.dtd_name) + assert_equal(document.render(), '\n

text

\n') + + +#################################################################################################################################### +def test_xml_parse(): + """Xml that cannot be parsed is reported with the name of the file, since the error is otherwise only a position.""" + + assert_equal(xml_parse("

text

", "test.xml").tag, "doc") + + with assert_raises(ToolError) as error: + xml_parse("

", "test.xml") + + assert_in("unable to parse 'test.xml':", str(error.exception)) + + +#################################################################################################################################### +def test_xml_attribute(): + """An attribute is read, set, and removed by name.""" + + node = xml_parse('', "test.xml") + + assert_equal(xml_node_attribute(node, "title"), "Reference") + assert_is_none(xml_node_attribute(node, "missing")) + + with assert_raises(ToolError) as error: + xml_node_attribute(node, "missing", True) + + assert_equal(str(error.exception), "unable to find attribute 'missing' in node 'doc'") + + xml_node_attribute_set(node, "title", "Other") + + assert_equal(xml_node_attribute(node, "title"), "Other") + + # Removing one that is there and one that is not, since a node that never had it is already how it should end up + xml_node_attribute_remove(node, "toc") + xml_node_attribute_remove(node, "missing") + + assert_equal(_render(node), '') + + +#################################################################################################################################### +def test_xml_child(): + """A child is found by name, and every child of a name can be found at once.""" + + node = xml_parse("

one

two

three", "test.xml") + + assert_equal(xml_node_content(xml_node_child(node, "p")), "one") + assert_equal([xml_node_content(child) for child in xml_node_child_list(node, "p")], ["one", "two"]) + assert_is_none(xml_node_child(node, "missing")) + + with assert_raises(ToolError) as error: + xml_node_child(node, "missing", True) + + assert_equal(str(error.exception), "unable to find child 'missing' in node 'doc'") + + # A node holds all the text under it, whatever markup it is spread across + node = xml_parse("

Run pgbackrest as root.

", "test.xml") + + assert_equal(xml_node_content(node), "Run pgbackrest as root.") + + +#################################################################################################################################### +def test_xml_dup(): + """A copy can be changed without changing what it came from, which is what makes a block reusable.""" + + node = xml_parse("

text

", "test.xml") + dup = xml_node_dup(node) + + xml_node_text_set(xml_node_child(dup, "p"), "other") + + assert_equal(_render(node), "

text

") + assert_equal(_render(dup), "

other

") + + +#################################################################################################################################### +def test_xml_content_add(): + """Text is added to the end of what a node holds, which is how a title of text and markup in turn is built.""" + + node = etree.Element("title") + + xml_node_content_add(node, "Backup Command (") + xml_node_content_add(xml_node_add(node, "id"), "backup") + xml_node_content_add(node, ")") + + assert_equal(_render(node), "Backup Command (<id>backup</id>)") + + # Text added to a node that already ends with text goes on the end of it rather than replacing it + xml_node_content_add(node, " and more") + + assert_equal(_render(node), "Backup Command (<id>backup</id>) and more") + + +#################################################################################################################################### +def test_xml_text_set(): + """Setting text replaces everything a node holds, unlike adding it.""" + + node = xml_parse("

Run pgbackrest now.

", "test.xml") + + xml_node_text_set(node, "Run it now.") + + assert_equal(_render(node), "

Run it now.

") + + +#################################################################################################################################### +def test_xml_child_add(): + """Everything a source node holds is copied, with the text and the markup left in the order they were written.""" + + node = etree.Element("p") + source = xml_parse('Run pgbackrest as root now.', "test.xml") + + xml_node_child_add(node, source) + + assert_equal(_render(node), '

Run pgbackrest as root now.

') + + # A comment is dropped by the parser, so it is never there to copy + node = etree.Element("p") + source = xml_parse("onetwo", "test.xml") + + assert_equal(len(source), 0) + + xml_node_child_add(node, source) + + assert_equal(_render(node), "

onetwo

") + + # Markup nested more than one deep + node = etree.Element("p") + source = xml_parse("one", "test.xml") + + xml_node_child_add(node, source) + + assert_equal(_render(node), "

one

") + + +#################################################################################################################################### +def test_xml_child_replace(): + """A node is replaced by what a source node holds, in the place the node was rather than at the end.""" + + # A node in the middle, with text on both sides of it + node = xml_parse("

before after

", "test.xml") + source = xml_parse("one two three", "test.xml") + + xml_node_child_replace(node, xml_node_child(node, "insert"), source) + + assert_equal(_render(node), "

before one two three after

") + + # The first node, so the text it begins with belongs to what holds it + node = xml_parse("

after

", "test.xml") + source = xml_parse("one two", "test.xml") + + xml_node_child_replace(node, xml_node_child(node, "insert"), source) + + assert_equal(_render(node), "

one two after

") + + # A node that follows another, so the text belongs to the one before it + node = xml_parse("

firstlast

", "test.xml") + source = xml_parse("middle", "test.xml") + + xml_node_child_replace(node, xml_node_child(node, "insert"), source) + + assert_equal(_render(node), "

firstmiddlelast

") + + # A source that holds nothing, which removes the node and leaves what was around it + node = xml_parse("

before after

", "test.xml") + source = xml_parse("", "test.xml") + + xml_node_child_replace(node, xml_node_child(node, "insert"), source) + + assert_equal(_render(node), "

before after

") + + # A source that holds nothing, replacing the first node + node = xml_parse("

after

", "test.xml") + source = xml_parse("", "test.xml") + + xml_node_child_replace(node, xml_node_child(node, "insert"), source) + + assert_equal(_render(node), "

after

") diff --git a/test/src/module/build/common/yaml_test.py b/test/src/module/build/common/yaml_test.py new file mode 100644 index 000000000..a60bd3b17 --- /dev/null +++ b/test/src/module/build/common/yaml_test.py @@ -0,0 +1,73 @@ +"""Test Yaml Handler. + +What matters here is what the loader does differently from a normal yaml load, since the generators depend on all three: scalars stay +text, mappings keep their order, and a repeated key keeps both values.""" + +#################################################################################################################################### +from harness.test import * + +from common.error import * +from common.yaml import * + + +#################################################################################################################################### +def test_yaml_load(): + """Every scalar is loaded as the text it was written as.""" + + value = yaml_load("version: 9.6\ncode: 011\nfatal: true\nempty:\n", "test.yaml") + + assert_equal([(key, value) for key, value in value], [("version", "9.6"), ("code", "011"), ("fatal", "true"), ("empty", "")]) + + # A sequence is a list and a mapping is not, so a range written as a pair can be told from a range written per map + value = yaml_load("range: [1, 2]\nmap:\n - key: value\n", "test.yaml") + pair_list = list(value) + + assert_equal(pair_list[0][1], ["1", "2"]) + assert_is_instance(pair_list[1][1][0], YamlMap) + + +#################################################################################################################################### +def test_yaml_load_order(): + """A mapping keeps the order it was written in and every one of a repeated key.""" + + value = yaml_load("b: 1\na: 2\nb: 3\n", "test.yaml") + + assert_equal([(key, value) for key, value in value], [("b", "1"), ("a", "2"), ("b", "3")]) + assert_equal(len(value), 3) + + +#################################################################################################################################### +def test_yaml_load_error(): + """A file that is not yaml is reported with the name of the file, since the error is otherwise only a line number.""" + + with assert_raises(ToolError) as error: + yaml_load("key: [\n", "test.yaml") + + assert_in("unable to parse 'test.yaml':", str(error.exception)) + + +#################################################################################################################################### +def test_yaml_bool(): + """A boolean must be written as true or false rather than any of the other spellings yaml allows.""" + + assert_true(yaml_bool("true", "key")) + assert_false(yaml_bool("false", "key")) + + with assert_raises(ToolError) as error: + yaml_bool("yes", "key") + + assert_equal(str(error.exception), "invalid boolean 'yes' for key") + + +#################################################################################################################################### +def test_yaml_map_empty(): + """A key that is a name in a list has an empty map under it and nothing else.""" + + yaml_map_empty(list(yaml_load("key: {}\n", "test.yaml"))[0][1], "key") + + # A key with nothing under it is a scalar rather than an empty map, so it does not count + for content in ("key:\n", "key: value\n", "key:\n other: value\n"): + with assert_raises(ToolError) as error: + yaml_map_empty(list(yaml_load(content, "test.yaml"))[0][1], "key") + + assert_equal(str(error.exception), "key must be an empty map") diff --git a/test/src/module/build/commonTest.c b/test/src/module/build/commonTest.c deleted file mode 100644 index 3effee3bd..000000000 --- a/test/src/module/build/commonTest.c +++ /dev/null @@ -1,86 +0,0 @@ -/*********************************************************************************************************************************** -Test Build Common -***********************************************************************************************************************************/ - -/*********************************************************************************************************************************** -Test Run -***********************************************************************************************************************************/ -static void -testRun(void) -{ - FUNCTION_HARNESS_VOID(); - - // ***************************************************************************************************************************** - if (testBegin("render")) - { - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("bldEnum()"); - - TEST_RESULT_STR_Z(bldEnum(NULL, STRDEF("test-id/group-name")), "testId/groupName", "enum"); - TEST_RESULT_STR_Z(bldEnum("pre", STRDEF("option-name")), "preOptionName", "enum"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("bldStrId*()"); - - TEST_RESULT_STR_Z(bldStrId("abc"), "STRID5(\"abc\", 0xc410)", "5-bit"); - TEST_RESULT_STR_Z(bldStrId("ABC"), "STRID6(\"ABC\", 0x289e61)", "6-bit"); - TEST_RESULT_STR_Z(bldStrIdSeq("ABC", 37), "STRID6S(\"ABC\", 37, 0x513cdff)", "6-bit with sequence"); - } - - // ***************************************************************************************************************************** - if (testBegin("Yaml")) - { - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("parse and error"); - - const Buffer *buffer = BUFSTRZ( - "test:\n" - " main: [0, 1]\n" - " default: text\n"); - - Yaml *yaml = NULL; - TEST_ASSIGN(yaml, yamlNew(buffer), "new yaml") - TEST_RESULT_VOID(yamlEventNextCheck(yaml, yamlEventTypeMapBegin), "map begin event"); - TEST_ERROR( - yamlScalarCheck(yamlEventPeek(yaml), STRDEF("xxx")), FormatError, - "expected scalar 'xxx' but got 'test' at line 1, column 1"); - TEST_ERROR( - yamlEventCheck(yamlEventPeek(yaml), yamlEventTypeMapBegin), FormatError, - "expected event type 'map-begin' but got scalar 'test' at line 1, column 1"); - TEST_RESULT_VOID(yamlScalarNextCheck(yaml, STRDEF("test")), "scalar event"); - TEST_ERROR( - yamlScalarCheck(yamlEventPeek(yaml), STRDEF("test")), FormatError, - "expected scalar 'test' but got event type 'map-begin' at line 2, column 4"); - TEST_RESULT_VOID(yamlEventNextCheck(yaml, yamlEventTypeMapBegin), "map begin event"); - TEST_RESULT_VOID(yamlEventNextCheck(yaml, yamlEventTypeScalar), "scalar event"); - TEST_RESULT_VOID(yamlEventNextCheck(yaml, yamlEventTypeSeqBegin), "seq begin event"); - TEST_RESULT_VOID(yamlEventNextCheck(yaml, yamlEventTypeScalar), "scalar event"); - TEST_RESULT_VOID(yamlEventNextCheck(yaml, yamlEventTypeScalar), "scalar event"); - TEST_RESULT_VOID(yamlEventNextCheck(yaml, yamlEventTypeSeqEnd), "seq end event"); - TEST_RESULT_VOID(yamlEventNextCheck(yaml, yamlEventTypeMapEnd), "map end event"); - TEST_ERROR( - yamlEventNextCheck(yaml, yamlEventTypeScalar), FormatError, - "yaml parse error: did not find expected key at line: 3 column: 3"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("boolean parse"); - - TEST_RESULT_BOOL(yamlBoolParse((YamlEvent){.value = STRDEF("true")}), true, "true"); - TEST_RESULT_BOOL(yamlBoolParse((YamlEvent){.value = STRDEF("false")}), false, "false"); - TEST_ERROR(yamlBoolParse((YamlEvent){.value = STRDEF("ack")}), FormatError, "invalid boolean 'ack'"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("type map (remaining types)"); - - TEST_RESULT_UINT(yamlEventType(YAML_STREAM_END_EVENT), yamlEventTypeStreamEnd, "stream end"); - TEST_RESULT_UINT(yamlEventType(YAML_DOCUMENT_END_EVENT), yamlEventTypeDocEnd, "doc end"); - TEST_RESULT_UINT(yamlEventType(YAML_ALIAS_EVENT), yamlEventTypeAlias, "alias"); - TEST_RESULT_UINT(yamlEventType(YAML_NO_EVENT), yamlEventTypeNone, "none"); - - TEST_ERROR( - yamlEventCheck((YamlEvent){.type = yamlEventTypeAlias}, yamlEventTypeScalar), FormatError, - "expected event type 'scalar' but got 'alias' at line 0, column 0"); - } - - FUNCTION_HARNESS_RETURN_VOID(); -} diff --git a/test/src/module/build/config/common_test.py b/test/src/module/build/config/common_test.py new file mode 100644 index 000000000..967a25019 --- /dev/null +++ b/test/src/module/build/config/common_test.py @@ -0,0 +1,61 @@ +"""Test Config Value Parsing. + +These must agree with the C, which parses the same values at run time, so the expectations here are the byte and millisecond counts the +C is known to produce rather than whatever this implementation happens to produce.""" + +#################################################################################################################################### +from harness.test import * + +from common.error import * +from config.common import * + + +#################################################################################################################################### +def test_cfg_parse_size(): + """A size is the number and the qualifier it is multiplied by.""" + + # No qualifier is bytes + assert_equal(cfg_parse_size("0"), 0) + assert_equal(cfg_parse_size("1024"), 1024) + + # Every qualifier, in each of the spellings that are allowed + assert_equal(cfg_parse_size("5b"), 5) + assert_equal(cfg_parse_size("1k"), 1024) + assert_equal(cfg_parse_size("1kb"), 1024) + assert_equal(cfg_parse_size("1kib"), 1024) + assert_equal(cfg_parse_size("1KiB"), 1024) + assert_equal(cfg_parse_size("1m"), 1048576) + assert_equal(cfg_parse_size("1g"), 1073741824) + assert_equal(cfg_parse_size("1t"), 1099511627776) + assert_equal(cfg_parse_size("1p"), 1125899906842624) + + for value in ("", "1x", "kb", "1.5g", "-1"): + with assert_raises(ToolError) as error: + cfg_parse_size(value) + + assert_equal(str(error.exception), "value '%s' is not valid" % value) + + +#################################################################################################################################### +def test_cfg_parse_time(): + """A time is the number in milliseconds, which is the unit the C keeps it in.""" + + # No qualifier is seconds + assert_equal(cfg_parse_time("0"), 0) + assert_equal(cfg_parse_time("30"), 30000) + + # Every qualifier, including the only one that is two characters + assert_equal(cfg_parse_time("100ms"), 100) + assert_equal(cfg_parse_time("100MS"), 100) + assert_equal(cfg_parse_time("30s"), 30000) + assert_equal(cfg_parse_time("15m"), 15 * 60 * 1000) + assert_equal(cfg_parse_time("1h"), 60 * 60 * 1000) + assert_equal(cfg_parse_time("1d"), 24 * 60 * 60 * 1000) + assert_equal(cfg_parse_time("1w"), 7 * 24 * 60 * 60 * 1000) + + # A qualifier that is not one, no number to go with the qualifier, and a number that is not one + for value in ("30x", "s", "", "1.5s"): + with assert_raises(ToolError) as error: + cfg_parse_time(value) + + assert_equal(str(error.exception), "value '%s' is not valid" % value) diff --git a/test/src/module/build/config/parse_test.py b/test/src/module/build/config/parse_test.py new file mode 100644 index 000000000..3ec85d663 --- /dev/null +++ b/test/src/module/build/config/parse_test.py @@ -0,0 +1,679 @@ +"""Test Configuration Parse. + +The declaration is written out here rather than read from the repository, since what needs testing is what a declaration means -- and +in particular what it inherits -- rather than what the project happens to declare today. + +Most tests share one declaration and check a different part of the result, so the parts that inherit from each other stay in one place +and it is clear what each option was declared against.""" + +#################################################################################################################################### +import os +import tempfile + +from harness.test import * + +from common.error import * +from common.storage import file_write +from config.parse import * + +# Commands the options are declared against. Backup sets everything a command can set, help and version take no options of their own, +# and restore takes only what a command gets by default. +COMMAND = """command: + backup: + command-role: + async: {} + local: {} + remote: {} + internal: true + lock-type: backup + lock-required: true + lock-remote-required: true + log-file: false + log-level-default: DETAIL + parameter-allowed: true + + help: {} + + restore: {} + + version: {} + +optionGroup: + pg: {} + repo: {} + +option: +""" + +# An option that everything else can be declared against +OPTION_STANZA = """ stanza: + type: string +""" + + +#################################################################################################################################### +def _parse(option, command=COMMAND): + """Parse a declaration built from the option section given.""" + + with tempfile.TemporaryDirectory() as path: + file_write(os.path.join(path, "build/config.yaml"), command + option) + + return bld_cfg_parse(path) + + +#################################################################################################################################### +def _error(option, command=COMMAND): + """Parse a declaration that is expected to fail and return the message.""" + + with assert_raises(ToolError) as error: + _parse(option, command) + + return str(error.exception) + + +#################################################################################################################################### +def _find(item_list, name): + """Find an entry in a list by name.""" + + return next(item for item in item_list if item.name == name) + + +#################################################################################################################################### +def test_config_parse_command(): + """A command takes what it declares, and every command has a main role whether it declares one or not.""" + + cfg = _parse(OPTION_STANZA) + + # Commands are sorted, whatever order they were declared in + assert_equal([cmd.name for cmd in cfg.cmd_list], ["backup", "help", "restore", "version"]) + assert_equal([opt_grp.name for opt_grp in cfg.opt_grp_list], ["pg", "repo"]) + + cmd = _find(cfg.cmd_list, "backup") + + assert_true(cmd.internal) + assert_equal(cmd.lock_type, "backup") + assert_true(cmd.lock_required) + assert_true(cmd.lock_remote_required) + assert_false(cmd.log_file) + assert_equal(cmd.log_level_default, "detail") + assert_true(cmd.parameter_allowed) + + # Main is added to the roles and the result is sorted + assert_equal(cmd.role_list, ["async", "local", "main", "remote"]) + + # A command that declares nothing gets the defaults, which includes writing to a log file + cmd = _find(cfg.cmd_list, "restore") + + assert_false(cmd.internal) + assert_equal(cmd.lock_type, "none") + assert_true(cmd.log_file) + assert_equal(cmd.log_level_default, "info") + assert_equal(cmd.role_list, ["main"]) + + +#################################################################################################################################### +def test_config_parse_command_error(): + """A command definition that is not one is reported rather than ignored.""" + + assert_equal( + _error(OPTION_STANZA, "command:\n backup:\n bogus: true\n\noptionGroup:\n pg: {}\n\noption:\n"), + "unknown command definition 'bogus'", + ) + + assert_equal( + _error(OPTION_STANZA, "command:\n backup:\n command-role:\n main: value\n\noptionGroup:\n pg: {}\n\noption:\n"), + "command 'backup' role 'main' must be an empty map", + ) + + assert_equal( + _error(OPTION_STANZA, "command:\n backup:\n internal: yes\n\noptionGroup:\n pg: {}\n\noption:\n"), + "invalid boolean 'yes' for command 'backup' internal", + ) + + assert_equal( + _error(OPTION_STANZA, "command:\n backup: {}\n\noptionGroup:\n pg: value\n\noption:\n"), + "option group 'pg' must be an empty map", + ) + + +#################################################################################################################################### +def test_config_parse_option(): + """An option takes what it declares, and what it does not declare follows from its type and section.""" + + cfg = _parse( + OPTION_STANZA + + """ buffer-size: + section: global + type: size + default: 1MiB + allow-range: [16KiB, 1GiB] + beta: true + secure: true + + force: + type: boolean + + online: + section: global + type: boolean + default: true + negate: false + + repo-cipher-type: + section: global + type: string-id + sequence: true + + start-fast: + section: global + type: boolean + bool-like: true + default-type: literal + + pg-path: + section: global + group: pg + type: path + required: false + deprecate: + db-path: {} + db?-path: {} +""" + ) + + # Options are sorted, whatever order they were declared in + assert_equal( + [opt.name for opt in cfg.opt_list], + ["buffer-size", "force", "online", "pg-path", "repo-cipher-type", "stanza", "start-fast"], + ) + + opt = _find(cfg.opt_list, "buffer-size") + + assert_equal(opt.type, "size") + assert_equal(opt.section, "global") + assert_equal(opt.default_value.value, "1MiB") + assert_equal(opt.default_type, "quote") + assert_equal((opt.allow_range.min, opt.allow_range.max), ("16KiB", "1GiB")) + assert_true(opt.beta) + assert_true(opt.secure) + + # An option in the configuration file can be reset, which is what removes it again + assert_true(opt.reset) + + # Only a boolean in the configuration file can be negated, unless the option says otherwise + assert_false(opt.negate) + assert_false(_find(cfg.opt_list, "force").negate) + assert_false(_find(cfg.opt_list, "online").negate) + + # An option that accepts y/n is treated as a boolean, so it can be negated too + opt = _find(cfg.opt_list, "start-fast") + + assert_true(opt.bool_like) + assert_true(opt.negate) + assert_equal(opt.default_type, "literal") + + # An option whose values are numbered + assert_true(_find(cfg.opt_list, "repo-cipher-type").sequence) + + # An option that does not say where it can be configured can only be given on the command line, so it cannot be reset + opt = _find(cfg.opt_list, "force") + + assert_equal(opt.section, "command-line") + assert_false(opt.reset) + assert_true(opt.required) + + # A command list that is not declared is every command but the ones that take no options + assert_equal([cmd.name for cmd in opt.cmd_list], ["backup", "restore"]) + + opt = _find(cfg.opt_list, "pg-path") + + assert_equal(opt.group, "pg") + assert_false(opt.required) + + # The same deprecated name written both ways is one deprecation that can be spelled either way + assert_equal([(dep.name, dep.indexed, dep.unindexed) for dep in opt.deprecate_list], [("db-path", True, True)]) + + +#################################################################################################################################### +def test_config_parse_option_error(): + """An option declaration that cannot be honored is reported rather than ignored.""" + + assert_equal(_error(OPTION_STANZA + " bogus:\n bogus: true\n"), "unknown option definition 'bogus'") + assert_equal(_error(OPTION_STANZA + " bogus:\n section: global\n"), "option 'bogus' requires 'type'") + assert_equal( + _error(OPTION_STANZA + " bogus:\n type: string\n default-type: bogus\n"), + "option 'bogus' has invalid default type 'bogus'", + ) + assert_equal( + _error(OPTION_STANZA + " bogus:\n type: string\n group: bogus\n"), "option 'bogus' has invalid group 'bogus'" + ) + assert_equal( + _error(OPTION_STANZA + " bogus:\n type: string\n command:\n bogus: {}\n"), + "invalid command 'bogus' in option 'bogus' command list", + ) + + # The stanza option must exist and must not depend on anything, since it is resolved before everything else + assert_equal(_error(" force:\n type: boolean\n"), "option 'stanza' must exist") + assert_equal( + _error(" force:\n type: boolean\n stanza:\n type: string\n depend:\n option: force\n"), + "option 'stanza' may not depend on other option", + ) + assert_equal( + _error( + " force:\n type: boolean\n" + " stanza:\n type: string\n command:\n backup:\n depend:\n option: force\n" + ), + "option 'stanza' command 'backup' may not depend on other option", + ) + + +#################################################################################################################################### +def test_config_parse_inherit(): + """An option can be declared against another option, which is what keeps a family of options in step.""" + + cfg = _parse( + OPTION_STANZA + + """ repo-path: + section: global + group: repo + type: path + default: /var/lib/pgbackrest + command: + backup: {} + deprecate: + repo-path: {} + + repo-host: + inherit: repo-path + type: string + default: ~ + + repo-host-user: + inherit: repo-host + default: pgbackrest + + repo-retention-full: + section: global + group: repo + type: integer + allow-list: + - 1 + - 2 + command: repo-path +""" + ) + + # Everything is inherited but what is declared after the inherit, and a tilde takes the default away again + opt = _find(cfg.opt_list, "repo-host") + + assert_equal(opt.type, "string") + assert_equal(opt.section, "global") + assert_equal(opt.group, "repo") + assert_is_none(opt.default_value) + assert_equal([cmd.name for cmd in opt.cmd_list], ["backup"]) + + # A deprecation names the option it is a deprecation of, so it is not inherited + assert_is_none(opt.deprecate_list) + assert_equal([dep.name for dep in _find(cfg.opt_list, "repo-path").deprecate_list], ["repo-path"]) + + # Inheriting from an option that itself inherited + opt = _find(cfg.opt_list, "repo-host-user") + + assert_equal(opt.type, "string") + assert_equal(opt.default_value.value, "pgbackrest") + + # A command list can be taken from another option on its own + assert_equal([cmd.name for cmd in _find(cfg.opt_list, "repo-retention-full").cmd_list], ["backup"]) + + +#################################################################################################################################### +def test_config_parse_inherit_error(): + """An option can only be declared against an option above it, since that is the order the declaration is read in.""" + + assert_equal( + _error(OPTION_STANZA + " bogus:\n inherit: later\n later:\n type: string\n"), + "option 'later' inherited before it is defined", + ) + assert_equal( + _error(OPTION_STANZA + " bogus:\n type: string\n command: later\n later:\n type: string\n"), + "command list inherited from option 'later' before it is defined", + ) + assert_equal( + _error(OPTION_STANZA + " bogus:\n type: string\n command:\n +inherit: later\n later:\n type: string\n"), + "commands inherited from option 'later' before it is defined", + ) + assert_equal( + _error(OPTION_STANZA + " bogus:\n type: string\n allow-list: later\n later:\n type: string\n"), + "allow list inherited from option 'later' before it is defined", + ) + assert_equal( + _error(OPTION_STANZA + " bogus:\n type: string\n depend: later\n later:\n type: string\n"), + "dependency inherited from option 'later' before it is defined", + ) + + +#################################################################################################################################### +def test_config_parse_allow(): + """The values an option allows can be listed, ranged, or taken from another option.""" + + cfg = _parse( + OPTION_STANZA + + """ repo-type: + section: global + type: string-id + default: posix + allow-list: + - posix + - s3 + - sftp: HAVE_LIBSSH2 + + repo-type-other: + section: global + type: string-id + allow-list: repo-type + + process-max: + section: global + type: integer + allow-range: [1, 999] + + compress-level: + section: global + type: integer + allow-range: + - none: [0, 0] + - gz: [0, 9] +""" + ) + + opt = _find(cfg.opt_list, "repo-type") + + assert_equal( + [(allow.value, allow.condition) for allow in opt.allow_list], [("posix", None), ("s3", None), ("sftp", "HAVE_LIBSSH2")] + ) + + # An allow list taken from another option is the same list + assert_equal([allow.value for allow in _find(cfg.opt_list, "repo-type-other").allow_list], ["posix", "s3", "sftp"]) + + # A range that maps on the values of another option + allow_range = _find(cfg.opt_list, "compress-level").allow_range + + assert_is_none(allow_range.min) + assert_equal([(map.map, map.min, map.max) for map in allow_range.map_list], [("none", "0", "0"), ("gz", "0", "9")]) + + +#################################################################################################################################### +def test_config_parse_allow_error(): + """A range or a conditional value that is not written as one is reported.""" + + assert_equal( + _error(OPTION_STANZA + " bogus:\n type: integer\n allow-range: [1, 2, 3]\n"), + "allow range for option 'bogus' must be a min and a max", + ) + assert_equal( + _error(OPTION_STANZA + " bogus:\n type: integer\n allow-range:\n - none: [0]\n"), + "allow range for option 'bogus' map 'none' must be a min and a max", + ) + assert_equal( + _error(OPTION_STANZA + " bogus:\n type: string-id\n allow-list:\n - a: X\n b: Y\n"), + "allow list value for option 'bogus' must have a single condition", + ) + + # An allow list cannot be inherited inside a command, where there is no option list to inherit from + assert_equal( + _error(OPTION_STANZA + " bogus:\n type: string-id\n command:\n backup:\n allow-list: stanza\n"), + "allow list for option 'bogus' command 'backup' cannot be inherited", + ) + + +#################################################################################################################################### +def test_config_parse_depend(): + """A dependency is on another option and on the values of it that make this option valid.""" + + cfg = _parse( + OPTION_STANZA + + """ online: + section: global + type: boolean + default: true + + archive-check: + section: global + type: boolean + default: true + depend: + option: online + default: false + list: + - true + + archive-copy: + section: global + type: boolean + depend: archive-check +""" + ) + + depend = _find(cfg.opt_list, "archive-check").depend + + # The dependency points at the option itself rather than at its name, so the render can reach what it needs + assert_equal(depend.option.name, "online") + assert_equal(depend.option.type, "boolean") + assert_equal(depend.default_value, "false") + assert_equal(depend.value_list, ["true"]) + + # A dependency taken from another option is the same dependency + assert_equal(_find(cfg.opt_list, "archive-copy").depend.option.name, "online") + + +#################################################################################################################################### +def test_config_parse_depend_error(): + """A dependency that cannot be honored is reported.""" + + assert_equal( + _error(OPTION_STANZA + " bogus:\n type: string\n depend:\n bogus: true\n"), "unknown depend definition 'bogus'" + ) + assert_equal( + _error(OPTION_STANZA + " bogus:\n type: string\n depend:\n option: missing\n"), + "dependency on undefined option 'missing'", + ) + + # Only an option the C can default without knowing the value can have a dependency default + assert_equal( + _error( + OPTION_STANZA + + " online:\n type: boolean\n" + + " bogus:\n type: string\n depend:\n option: online\n default: x\n" + ), + "dependency default invalid for non integer/boolean option 'bogus'", + ) + + +#################################################################################################################################### +def test_config_parse_option_command(): + """What an option means for a command is what it declares for the command falling back to what it declares for itself.""" + + cfg = _parse( + OPTION_STANZA + + """ set: + type: string + internal: true + required: false + command: + +role: any + -command: restore + -command: missing + backup: + internal: false + required: true + sequence: true + default: latest + allow-list: + - latest + command-role: + main: {} + local: {} + + type: + type: string + command: + +inherit: set + +role: local +""" + ) + + opt = _find(cfg.opt_list, "set") + + # Every command that has any role, less the ones removed, and never the commands that take no options + assert_equal([cmd.name for cmd in opt.cmd_list], ["backup"]) + + cmd = _find(opt.cmd_list, "backup") + + # What the command declares wins over what the option declares + assert_false(cmd.internal) + assert_true(cmd.required) + assert_true(cmd.sequence) + assert_equal(cmd.default_value.value, "latest") + assert_equal([allow.value for allow in cmd.allow_list], ["latest"]) + + # A role list declared for the command is the command's own + assert_equal(cmd.role_list, ["main", "local"]) + + # A command that takes its own roles from the option keeps only the roles that command has + opt = _find(cfg.opt_list, "type") + + assert_equal([cmd.name for cmd in opt.cmd_list], ["backup"]) + assert_equal(_find(opt.cmd_list, "backup").role_list, ["async", "local", "main", "remote"]) + + # What the command does not declare follows the option + cmd = _find(opt.cmd_list, "backup") + + assert_false(cmd.internal) + assert_true(cmd.required) + assert_false(cmd.sequence) + + +#################################################################################################################################### +def test_config_parse_option_command_role(): + """An option can name the roles it is valid for, which are then filtered against the roles each command has.""" + + cfg = _parse( + OPTION_STANZA + + """ process: + type: integer + command-role: + local: {} + remote: {} + command: + backup: {} + restore: {} +""" + ) + + opt = _find(cfg.opt_list, "process") + + # Backup has both roles and restore has neither, so restore is left with nothing + assert_equal(_find(opt.cmd_list, "backup").role_list, ["local", "remote"]) + assert_equal(_find(opt.cmd_list, "restore").role_list, []) + + +#################################################################################################################################### +def test_config_parse_option_command_error(): + """A command definition inside an option that is not one is reported.""" + + assert_equal( + _error(OPTION_STANZA + " bogus:\n type: string\n command:\n backup:\n bogus: true\n"), + "unknown option command definition 'bogus'", + ) + + +#################################################################################################################################### +def test_config_parse_default(): + """A default can be one value or a value per value of the option it maps on.""" + + cfg = _parse( + OPTION_STANZA + + """ repo-type: + section: global + type: string-id + default: posix + + repo-path: + section: global + type: path + default: + - posix: /var/lib/pgbackrest + - s3: / +""" + ) + + default = _find(cfg.opt_list, "repo-path").default_value + + assert_is_none(default.value) + assert_equal([(map.map, map.value) for map in default.map_list], [("posix", "/var/lib/pgbackrest"), ("s3", "/")]) + + +#################################################################################################################################### +def test_config_parse_resolve(): + """The options are ordered so that every option comes after the options it depends on.""" + + cfg = _parse( + OPTION_STANZA + + """ online: + section: global + type: boolean + + archive-check: + section: global + type: boolean + depend: + option: online + + archive-copy: + section: global + type: boolean + command: + backup: + depend: + option: archive-check +""" + ) + + # Stanza is always first, since an error about a missing option is confusing when the stanza is what is missing. After that an + # option follows whatever it depends on, whether the dependency is its own or one of its commands. + assert_equal([opt.name for opt in cfg.opt_resolve_list], ["stanza", "online", "archive-check", "archive-copy"]) + + +#################################################################################################################################### +def test_config_parse_resolve_error(): + """Options that depend on each other cannot be ordered and are reported together.""" + + assert_equal( + _error( + OPTION_STANZA + + " a:\n type: boolean\n depend:\n option: b\n" + + " b:\n type: boolean\n depend:\n option: a\n" + ), + "unable to resolve dependencies for option(s) 'a, b'\nHINT: are there circular dependencies?", + ) + + +#################################################################################################################################### +def test_config_parse_error(): + """A declaration that is missing a section, or has one that is not expected, is reported.""" + + with tempfile.TemporaryDirectory() as path: + file_write(os.path.join(path, "build/config.yaml"), "bogus:\n key: value\n") + + with assert_raises(ToolError) as error: + bld_cfg_parse(path) + + assert_equal(str(error.exception), "unknown config definition 'bogus'") + + with tempfile.TemporaryDirectory() as path: + file_write(os.path.join(path, "build/config.yaml"), "command:\n backup: {}\n") + + with assert_raises(ToolError) as error: + bld_cfg_parse(path) + + assert_equal(str(error.exception), "command, optionGroup, and option are required") diff --git a/test/src/module/build/config/render_test.py b/test/src/module/build/config/render_test.py new file mode 100644 index 000000000..b496cf04d --- /dev/null +++ b/test/src/module/build/config/render_test.py @@ -0,0 +1,638 @@ +"""Test Configuration Render. + +The declaration written here is a small one that still has an option of every type, since the value tables the rules index into are +built per type and a type with no values in it would not be rendered at all. + +The rules are checked as the fragments that matter rather than as whole files, since the whole of the real parse rules is a hundred +thousand lines. What guarantees the rest is that the same generator reproduces the checked-in files byte for byte.""" + +#################################################################################################################################### +import os +import tempfile + +from harness.test import * + +from common.error import * +from common.storage import file_read, file_write +from config.parse import bld_cfg_parse +from config.render import * + +COMMAND = """command: + backup: + command-role: + async: {} + local: {} + remote: {} + internal: true + lock-type: backup + lock-required: true + lock-remote-required: true + log-file: false + log-level-default: detail + parameter-allowed: true + + help: {} + + restore: {} + + version: {} + +optionGroup: + pg: {} + repo: {} + +option: +""" + +OPTION = """ annotation: + section: global + type: hash + + archive-check: + section: global + type: boolean + default: true + depend: + option: online + default: false + list: + - true + + archive-copy: + section: global + type: boolean + default: false + command: + backup: + default: true + depend: + option: archive-check + + buffer-size: + section: global + type: size + default: 1MiB + allow-range: [16KiB, 1GiB] + + compress-level: + section: global + type: integer + default-type: dynamic + default: compress-level + allow-range: + - none: [0, 0] + - gz: [0, 9] + + db-include: + section: global + type: list + + force: + type: boolean + internal: true + beta: true + secure: true + command: + backup: + internal: false + required: false + restore: + internal: false + required: false + + io-timeout: + section: global + type: time + default: 60 + allow-range: [100ms, 1h] + + online: + section: global + type: boolean + default: true + + pg-path: + section: global + group: pg + type: path + deprecate: + pg-path: {} + db?-path: {} + + process-max: + section: global + type: integer + default: 1 + allow-range: [1, 999] + + repo-cipher-type: + section: global + group: repo + type: string-id + sequence: true + default: aes-256-cbc + allow-list: + - none + - aes-256-cbc + + repo-host: + section: global + group: repo + type: string + + repo-host-user: + section: global + group: repo + type: string + default: pgbackrest + depend: + option: repo-host + + repo-path: + section: global + group: repo + type: path + default: /var/lib/pgbackrest + + repo-storage-port: + section: global + group: repo + type: integer + default: + - s3: 443 + - azure: 443 + + repo-type: + section: global + group: repo + type: string-id + default: posix + allow-list: + - posix + - posix + - s3 + - sftp: HAVE_LIBSSH2 + command: + backup: + allow-list: + - posix + - azure + + spool-path: + section: global + type: path + default-type: literal + default: PROJECT_PATH + + stanza: + type: string + + start-fast: + section: global + type: boolean + bool-like: true + default: false + + target: + type: string + command: + restore: {} + + target-action: + section: global + type: string-id + sequence: true + default: pause + allow-list: + - pause + - promote + + repo-storage-host: + section: global + group: repo + type: string + default: this-is-a-very-long-default-value-that-leaves-no-room-for-a-label-on-the-line-it-is-rendered-on + + tls-version: + section: global + type: string + default: any version + allow-list: + - "1.2" + - "1.3" + + type: + type: string-id + default: incr + allow-list: + - full + - diff + - incr + command: + backup: + required: false + restore: + sequence: true + default: default + allow-list: + - default + - immediate +""" + + +#################################################################################################################################### +def _render(option=OPTION, label=False): + """Render a declaration and return both generated files. + + Labels are off unless a test is checking them, since a label is appended to every line and would be in the way of reading the + rules the rest of the tests check.""" + + with tempfile.TemporaryDirectory() as path: + file_write(os.path.join(path, "build/config.yaml"), COMMAND + option) + bld_cfg_render(path, bld_cfg_parse(path), label) + + return ( + file_read(os.path.join(path, "src/config/config.auto.h")), + file_read(os.path.join(path, "src/config/parse.auto.c.inc")), + ) + + +#################################################################################################################################### +def test_config_render_constant(): + """A command, an option, and an option value each get the constant the code refers to it by.""" + + config, _ = _render() + + assert_in('#define CFGCMD_BACKUP "backup"\n', config) + assert_in("#define CFG_COMMAND_TOTAL 4\n", config) + assert_in("#define CFG_OPTION_GROUP_TOTAL 2\n", config) + + # An option in a group is referred to by the group and an index rather than by name, so it has no constant + assert_in('#define CFGOPT_STANZA "stanza"\n', config) + assert_not_in("CFGOPT_REPO_PATH ", config) + assert_in("#define CFG_OPTION_TOTAL 25\n", config) + + +#################################################################################################################################### +def test_config_render_value_constant(): + """A value of a string-id option gets the StringId it encodes to and the string it spells.""" + + config, _ = _render() + + assert_in('#define CFGOPTVAL_REPO_TYPE_POSIX_Z "posix"\n', config) + assert_in('#define CFGOPTVAL_REPO_TYPE_POSIX STRID5("posix", 0x', config) + + # An option whose values are numbered also gets the number, and the StringId carries it + assert_in("#define CFGOPTVAL_REPO_CIPHER_TYPE_NONE 0\n", config) + assert_in('#define CFGOPTVAL_REPO_CIPHER_TYPE_NONE_STRID STRID5S("none", 0, 0x', config) + + # Values an option allows only for one command are named after the command as well + assert_in("#define CFGOPTVAL_RESTORE_TYPE_DEFAULT 0\n", config) + assert_in("#define CFGOPTVAL_RESTORE_TYPE_IMMEDIATE 1\n", config) + + +#################################################################################################################################### +def test_config_render_enum(): + """Every command, option group, and option is in the enum the rules index it by.""" + + config, _ = _render() + + assert_in( + """typedef enum +{ + cfgCmdBackup, + cfgCmdHelp, + cfgCmdRestore, + cfgCmdVersion, +} ConfigCommand; +""", + config, + ) + + assert_in( + """typedef enum +{ + cfgOptGrpPg, + cfgOptGrpRepo, +} ConfigOptionGroup; +""", + config, + ) + + assert_in(" cfgOptRepoCipherType,\n", config) + + +#################################################################################################################################### +def test_config_render_command(): + """A command renders what it declares, and only what differs from the default.""" + + _, parse = _render() + + assert_in( + """ PARSE_RULE_COMMAND + ( + PARSE_RULE_COMMAND_NAME("backup"),""", + parse, + ) + + for rule in ( + "PARSE_RULE_COMMAND_INTERNAL(true)", + "PARSE_RULE_COMMAND_LOCK_REQUIRED(true),", + "PARSE_RULE_COMMAND_LOCK_REMOTE_REQUIRED(true),", + "PARSE_RULE_COMMAND_LOCK_TYPE(Backup),", + "PARSE_RULE_COMMAND_LOG_LEVEL_DEFAULT(Detail),", + "PARSE_RULE_COMMAND_PARAMETER_ALLOWED(true),", + "PARSE_RULE_COMMAND_ROLE(Async)", + ): + assert_in(rule, parse) + + # A command that writes to a log file says so, and backup does not + assert_in("PARSE_RULE_COMMAND_LOG_FILE(true),", parse) + + # An option group is rendered by name, since the code refers to a group by its enum + assert_in(' PARSE_RULE_OPTION_GROUP_NAME("pg"),', parse) + + +#################################################################################################################################### +def test_config_render_option(): + """An option renders what it declares, and only what differs from the default.""" + + _, parse = _render() + + for rule in ( + 'PARSE_RULE_OPTION_NAME("force"),', + "PARSE_RULE_OPTION_TYPE(Boolean),", + "PARSE_RULE_OPTION_INTERNAL(true)", + "PARSE_RULE_OPTION_BETA(true),", + "PARSE_RULE_OPTION_SECURE(true),", + "PARSE_RULE_OPTION_REQUIRED(true),", + "PARSE_RULE_OPTION_SECTION(CommandLine),", + "PARSE_RULE_OPTION_SEQUENCE(true),", + "PARSE_RULE_OPTION_NEGATE(true),", + "PARSE_RULE_OPTION_RESET(true),", + "PARSE_RULE_OPTION_GROUP_ID(Repo),", + "PARSE_RULE_OPTION_DEFAULT_TYPE(Dynamic),", + ): + assert_in(rule, parse) + + # A hash or a list can be given more than once, which builds the value up rather than replacing it + assert_in("PARSE_RULE_OPTION_MULTI(true),", parse) + + # An option that accepts y/n is treated as a boolean + assert_in("PARSE_RULE_OPTION_BOOL_LIKE(true),", parse) + + # An option in a group whose deprecated name is the option name can be given without an index + assert_in("PARSE_RULE_OPTION_DEPRECATE_MATCH(true),", parse) + + # Commands the option is internal for, when that differs from the option itself + assert_in( + """ PARSE_RULE_OPTION_COMMAND_INTERNAL_LIST + ( + PARSE_RULE_OPTION_COMMAND_INTERNAL(Backup, false), + PARSE_RULE_OPTION_COMMAND_INTERNAL(Restore, false), + )""", + parse, + ) + + # Commands the option is valid for, per role + assert_in("PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST", parse) + assert_in("PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST", parse) + + +#################################################################################################################################### +def test_config_render_optional(): + """A rule that applies only to some commands is rendered with a filter naming them.""" + + _, parse = _render() + + # Two commands whose rules came out the same share one group + assert_in( + """ PARSE_RULE_OPTIONAL_GROUP + ( + PARSE_RULE_FILTER_CMD + ( + PARSE_RULE_VAL_CMD(Backup), + PARSE_RULE_VAL_CMD(Restore), + ), + + PARSE_RULE_OPTIONAL_NOT_REQUIRED(), + ),""", + parse, + ) + + # A dependency names the option it is on and the values of it that make this option valid + assert_in( + """ PARSE_RULE_OPTIONAL_DEPEND + ( + PARSE_RULE_OPTIONAL_DEPEND_DEFAULT(PARSE_RULE_VAL_BOOL_FALSE), + PARSE_RULE_VAL_OPT(Online), + PARSE_RULE_VAL_BOOL_TRUE, + )""", + parse, + ) + + # A dependency on a string option is compared as a StringId, since it is on one of a small set of known values + assert_in("PARSE_RULE_VAL_OPT(RepoHost),", parse) + + # A range, either one range or a range per value of the option it maps on + assert_in( + """ PARSE_RULE_OPTIONAL_ALLOW_RANGE + ( + PARSE_RULE_VAL_INT(1), + PARSE_RULE_VAL_INT(999), + )""", + parse, + ) + + assert_in( + """ PARSE_RULE_OPTIONAL_ALLOW_RANGE_MAP + ( + PARSE_RULE_VAL_STRID(none), + PARSE_RULE_VAL_INT(0), + PARSE_RULE_VAL_INT(0), + PARSE_RULE_VAL_STRID(gz), + PARSE_RULE_VAL_INT(0), + PARSE_RULE_VAL_INT(9), + ),""", + parse, + ) + + # A value only compiled in with a feature is replaced by false when the feature is not, so it keeps its index + assert_in( + """ PARSE_RULE_VAL_STRID(sftp), +#ifndef HAVE_LIBSSH2 + PARSE_RULE_BOOL_FALSE, +#endif""", + parse, + ) + + # A default that is worked out at run time names what works it out rather than a value + assert_in("PARSE_RULE_DEFAULT_DYNAMIC(CompressLevel),", parse) + + # A default per value of the option it maps on + assert_in( + """ PARSE_RULE_OPTIONAL_DEFAULT_MAP + ( + PARSE_RULE_VAL_STRID(s3), + PARSE_RULE_VAL_INT(443), + PARSE_RULE_VAL_STRID(azure), + PARSE_RULE_VAL_INT(443), + ),""", + parse, + ) + + # A default that is not the first value the option allows also says where it sits, since that is what a sequence counts. A default + # that is the first needs nothing, since that is where a sequence starts. + assert_in("PARSE_RULE_VAL_SEQ(1),", parse) + assert_not_in("PARSE_RULE_VAL_SEQ(0),", parse) + + # A command that numbers the values differently from the option + assert_in("PARSE_RULE_OPTIONAL_SEQUENCE()", parse) + + +#################################################################################################################################### +def test_config_render_value(): + """Every value the rules refer to is in the table for its type, sorted by what it means.""" + + _, parse = _render() + + # A string is stored once and referred to by index, and a StringId is the string it spells + assert_in("#define PARSE_RULE_VAL_STR(value)", parse) + assert_in("#define PARSE_RULE_VAL_STRID(value) PARSE_RULE_VAL_STR(QT_##value##_QT)\n", parse) + + # A value that holds characters a name cannot has them spelled out + assert_in("parseRuleValStrQT_FS_var_FS_lib_FS_pgbackrest_QT,", parse) + assert_in("parseRuleValStrQT_1_DT_2_QT,", parse) + assert_in("parseRuleValStrQT_any_SP_version_QT,", parse) + + # A default written as the literal C to use is not quoted + assert_in("parseRuleValStrPROJECT_PATH,", parse) + + # Sizes and times are sorted by what they mean rather than by how they are written, and each keeps the text it was written as + assert_in( + """static const int64_t parseRuleValueSize[] = +{ + 16384,""", + parse, + ) + assert_in(" parseRuleValStrQT_16KiB_QT,", parse) + assert_in( + """static const unsigned int parseRuleValueTime[] = +{ + 100,""", + parse, + ) + + # Integers are sorted numerically and named after the value + assert_in(" parseRuleValInt999,", parse) + + # The order the options are resolved in, which is what the parse follows + assert_in("static const uint8_t optionResolveOrder[] =\n{\n cfgOptStanza,", parse) + + # Deprecations, which are the old names an option can still be given by + assert_in("#define CFG_OPTION_DEPRECATE_TOTAL 2\n", parse) + assert_in( + """ .name = "db-path", + .id = cfgOptPgPath, + .indexed = true,""", + parse, + ) + assert_in( + """ .name = "pg-path", + .id = cfgOptPgPath, + .unindexed = true,""", + parse, + ) + + +#################################################################################################################################### +def test_config_render_index(): + """A rule stores an index as a variable length integer, so an index too wide for one byte takes two.""" + + # An option list short enough that every index fits one byte + _, parse = _render() + + assert_in("#define PARSE_RULE_VAL_OPT(value) PARSE_RULE_U32_1(cfgOpt##value)\n", parse) + + # An option list long enough that it does not + option = OPTION + "".join(" extra-%03u:\n section: global\n type: boolean\n" % idx for idx in range(130)) + _, parse = _render(option) + + assert_in("#define PARSE_RULE_VAL_OPT(value) PARSE_RULE_U32_2(cfgOpt##value)\n", parse) + + +#################################################################################################################################### +def test_config_render_label(): + """Each rule is labelled with what it came from, so the generated file can be read.""" + + _, parse = _render(label=True) + + for line in parse.split("\n"): + if line.startswith(' PARSE_RULE_OPTION_NAME("force")'): + # The label sits at the right margin, which is the width the rest of the source uses + assert_equal(len(line), 132) + assert_true(line.endswith("// opt/force")) + + break + else: + raise ToolError("unable to find the option to check the label of") + + # A line already too long for the comment is left alone rather than wrapped + assert_in(" PARSE_RULE_VAL_STR(QT_this_DS_is_DS_a_DS_very_DS_long", parse) + + # Without labels every line is the rule alone + _, parse = _render() + + assert_not_in("// opt/force", parse) + + +#################################################################################################################################### +def test_config_render_error(): + """A sequence whose default is not one of the values it numbers cannot be rendered.""" + + with assert_raises(ToolError) as error: + _render( + """ stanza: + type: string + + start-fast: + section: global + type: boolean + bool-like: true + default: false + + target: + type: string + command: + restore: {} + + target-action: + section: global + type: string-id + sequence: true + default: pause + allow-list: + - pause + - promote + + repo-cipher-type: + section: global + type: string-id + sequence: true + default: bogus + allow-list: + - none +""" + ) + + assert_equal(str(error.exception), "unable to find default 'bogus' in allow list") + + +#################################################################################################################################### +def test_config_render_value_error(): + """A declaration with no value of a type cannot be rendered, since the array the rules index into cannot be empty.""" + + with assert_raises(ToolError) as error: + _render(" stanza:\n type: string\n") + + assert_equal(str(error.exception), "declaration has no string value for the rules to index") diff --git a/test/src/module/build/configTest.c b/test/src/module/build/configTest.c deleted file mode 100644 index 6f12357d3..000000000 --- a/test/src/module/build/configTest.c +++ /dev/null @@ -1,1930 +0,0 @@ -/*********************************************************************************************************************************** -Test Build Config -***********************************************************************************************************************************/ -#include "harness/storage.h" - -/*********************************************************************************************************************************** -Test Run -***********************************************************************************************************************************/ -static void -testRun(void) -{ - FUNCTION_HARNESS_VOID(); - - // Create default storage object for testing - Storage *storageTest = storagePosixNewP(TEST_PATH_STR, .write = true); - - // ***************************************************************************************************************************** - if (testBegin("bldCfgParse() and bldCfgRender()")) - { - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("render with label"); - - TEST_RESULT_STR_Z( - bldCfgRenderLabel(STRDEF("line"), true, STRDEF("label")), - "line " - " // label", - "render"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("render without label due to long line"); - - TEST_RESULT_STR_Z( - bldCfgRenderLabel( - STRDEF( - "12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890" - "12345678901234567890"), - true, STRDEF("label")), - "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678" - "901234567890", - "render"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("command parse errors"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/config/config.yaml", - "command:\n" - " archive-get:\n" - " bogus: test\n"); - - TEST_ERROR(bldCfgParse(storageTest), FormatError, "unknown command definition 'bogus'"); - - #define TEST_COMMAND_VALID \ - "command:\n" \ - " archive-get:\n" \ - " internal: true\n" \ - "\n" - - #define TEST_OPTION_GROUP_VALID \ - "optionGroup:\n" \ - " repo: {}\n" \ - "\n" - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("option parse errors"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/config/config.yaml", - TEST_COMMAND_VALID - TEST_OPTION_GROUP_VALID - "option:\n" - " config:\n" - " bogus: test\n"); - - TEST_ERROR(bldCfgParse(storageTest), FormatError, "unknown option definition 'bogus'"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/config/config.yaml", - TEST_COMMAND_VALID - TEST_OPTION_GROUP_VALID - "option:\n" - " config:\n" - " section: global\n"); - - TEST_ERROR(bldCfgParse(storageTest), FormatError, "option 'config' requires 'type'"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/config/config.yaml", - TEST_COMMAND_VALID - TEST_OPTION_GROUP_VALID - "option:\n" - " config:\n" - " depend:\n" - " bogus: test\n"); - - TEST_ERROR(bldCfgParse(storageTest), FormatError, "unknown depend definition 'bogus'"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/config/config.yaml", - TEST_COMMAND_VALID - TEST_OPTION_GROUP_VALID - "option:\n" - " config:\n" - " default-type: bogus\n"); - - TEST_ERROR(bldCfgParse(storageTest), FormatError, "option 'config' has invalid default type 'bogus'"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/config/config.yaml", - TEST_COMMAND_VALID - TEST_OPTION_GROUP_VALID - "option:\n" - " config:\n" - " command:\n" - " backup:\n" - " bogus: test\n"); - - TEST_ERROR(bldCfgParse(storageTest), FormatError, "unknown option command definition 'bogus'"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/config/config.yaml", - TEST_COMMAND_VALID - TEST_OPTION_GROUP_VALID - "option:\n" - " config:\n" - " depend: bogus\n"); - - TEST_ERROR(bldCfgParse(storageTest), FormatError, "dependency inherited from option 'bogus' before it is defined"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/config/config.yaml", - TEST_COMMAND_VALID - TEST_OPTION_GROUP_VALID - "option:\n" - " config:\n" - " type: string\n" - " command:\n" - " bogus: {}\n"); - - TEST_ERROR(bldCfgParse(storageTest), FormatError, "invalid command 'bogus' in option 'config' command list"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/config/config.yaml", - TEST_COMMAND_VALID - TEST_OPTION_GROUP_VALID - "option:\n" - " config:\n" - " type: string\n" - " depend:\n" - " option: bogus\n"); - - TEST_ERROR(bldCfgParse(storageTest), FormatError, "dependency on undefined option 'bogus'"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/config/config.yaml", - TEST_COMMAND_VALID - TEST_OPTION_GROUP_VALID - "option:\n" - " config:\n" - " type: string\n" - " group: bogus\n"); - - TEST_ERROR(bldCfgParse(storageTest), FormatError, "option 'config' has invalid group 'bogus'"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/config/config.yaml", - TEST_COMMAND_VALID - TEST_OPTION_GROUP_VALID - "option:\n" - " stanza:\n" - " type: string\n" - " config:\n" - " type: string\n" - " depend:\n" - " option: online\n" - " online:\n" - " type: boolean\n" - " depend:\n" - " option: config\n"); - - TEST_ERROR( - bldCfgParse(storageTest), FormatError, - "unable to resolve dependencies for option(s) 'config, online'\n" - "HINT: are there circular dependencies?"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/config/config.yaml", - TEST_COMMAND_VALID - TEST_OPTION_GROUP_VALID - "option:\n" - " config:\n" - " type: string\n"); - - TEST_ERROR(bldCfgParse(storageTest), FormatError, "option 'stanza' must exist"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/config/config.yaml", - TEST_COMMAND_VALID - TEST_OPTION_GROUP_VALID - "option:\n" - " config:\n" - " type: string\n" - " stanza:\n" - " type: string\n" - " depend:\n" - " option: config\n"); - - TEST_ERROR(bldCfgParse(storageTest), FormatError, "option 'stanza' may not depend on other option"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/config/config.yaml", - TEST_COMMAND_VALID - TEST_OPTION_GROUP_VALID - "option:\n" - " config:\n" - " type: string\n" - " stanza:\n" - " type: string\n" - " command:\n" - " archive-get:\n" - " depend:\n" - " option: config\n"); - - TEST_ERROR(bldCfgParse(storageTest), FormatError, "option 'stanza' command 'archive-get' may not depend on other option"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/config/config.yaml", - TEST_COMMAND_VALID - TEST_OPTION_GROUP_VALID - "option:\n" - " config:\n" - " type: bool\n" - " stanza:\n" - " type: string\n" - " command:\n" - " archive-get:\n" - " depend:\n" - " option: config\n" - " default: false\n"); - - TEST_ERROR(bldCfgParse(storageTest), FormatError, "dependency default invalid for non integer/boolean option 'stanza'"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("parse and render config"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/config/config.yaml", - "command:\n" - " archive-get:\n" - " command-role:\n" - " async: {}\n" - " local: {}\n" - " remote: {}\n" - " lock-type: archive\n" - " log-file: false\n" - " log-level-default: debug\n" - " parameter-allowed: true\n" - "\n" - " backup:\n" - " internal: true\n" - " command-role:\n" - " local: {}\n" - " remote: {}\n" - " lock-type: backup\n" - " lock-required: true\n" - " lock-remote-required: true\n" - "\n" - " help: {}\n" - "\n" - " version: {}\n" - "\n" - "optionGroup:\n" - " pg: {}\n" - " repo: {}\n" - "\n" - "option:\n" - " timeout:\n" - " type: time\n" - " default: 10s\n" - " command:\n" - " backup:\n" - " default: 100ms\n" - " archive-get:\n" - " default: 30m\n" - " allow-range: [100ms, 7w]\n" - "\n" - " buffer-size:\n" - " section: global\n" - " type: size\n" - " command:\n" - " backup: {}\n" - " archive-get:\n" - " default: 32KiB\n" - " allow-list:\n" - " - 32KiB\n" - " allow-list:\n" - " - 8KiB\n" - " - 16KiB: HAVE_LARGE_MEM\n" - " command-role:\n" - " main: {}\n" - "\n" - " cmd:\n" - " section: global\n" - " type: string\n" - " default-type: dynamic\n" - " default: bin\n" - " command:\n" - " backup: {}\n" - "\n" - " compress-network:\n" - " section: global\n" - " type: string-id\n" - " command:\n" - " backup:\n" - " allow-list:\n" - " - gz\n" - " archive-get:\n" - " sequence: true\n" - " allow-list:\n" - " - bz2\n" - " command-role:\n" - " main: {}\n" - "\n" - " compress-type:\n" - " section: global\n" - " type: string\n" - " sequence: true\n" - " default: gz\n" - " allow-list:\n" - " - none\n" - " - bz2\n" - " - gz\n" - " command:\n" - " backup: {}\n" - " archive-get:\n" - " depend:\n" - " option: config\n" - " command-role: {}\n" - " deprecate:\n" - " compress: {}\n" - "\n" - " compress-level:\n" - " section: global\n" - " type: integer\n" - " required: false\n" - " default:\n" - " - bz2: 9\n" - " - gz: 6\n" - " allow-range:\n" - " - bz2: [1, 9]\n" - " - gz: [-1, 9]\n" - " depend:\n" - " option: compress-type\n" - " default: 0\n" - " list:\n" - " - bz2\n" - " - gz\n" - " command: compress-type\n" - "\n" - " compress-level-network:\n" - " inherit: compress-level\n" - " internal: true\n" - " secure: true\n" - " default: ~\n" - " depend:\n" - " option: compress-type\n" - " list:\n" - " - none\n" - " - gz\n" - "\n" - " config:\n" - " type: string\n" - " default: CFGOPTDEF_CONFIG_PATH \"/\" PROJECT_CONFIG_FILE\n" - " default-type: literal\n" - " negate: true\n" - " command:\n" - " backup: {}\n" - " archive-get:\n" - " default: CFGOPTDEF_CONFIG_PATH \"/.-\" PROJECT_CONFIG_FILE\n" - "\n" - " config-include:\n" - " section: global\n" - " type: path\n" - " default-type: quote\n" - " default: /include\n" - " command:\n" - " +role: local\n" - " +role: remote\n" - " command-role:\n" - " main: {}\n" - "\n" - " log-level-console:\n" - " section: global\n" - " type: string-id\n" - " default: warn\n" - " allow-list:\n" - " - off\n" - " - error\n" - " - warn\n" - " - debug1\n" - "\n" - " log-level-file:\n" - " section: global\n" - " type: string-id\n" - " default: info\n" - " allow-list: log-level-console\n" - " command:\n" - " backup:\n" - " internal: true\n" - " required: false\n" - " default: warn\n" - " allow-list:\n" - " - off\n" - " - warn\n" - " depend:\n" - " option: log-level-console\n" - " list:\n" - " - warn\n" - " command-role:\n" - " main: {}\n" - " help: {}\n" - " archive-get:\n" - " default: ~\n" - "\n" - " stanza:\n" - " type: list\n" - " command:\n" - " archive-get: {}\n" - " backup: {}\n" - "\n" - " online:\n" - " type: boolean\n" - " default: true\n" - " command:\n" - " +inherit: log-level-file\n" - " -command: help\n" - " backup:\n" - " default: false\n" - " command-role:\n" - " main: {}\n" - " deprecate:\n" - " online-old: {}\n" - "\n" - " bool-like:\n" - " section: global\n" - " type: string-id\n" - " sequence: true\n" - " bool-like: true\n" - " default: n\n" - " allow-list:\n" - " - n\n" - " - check\n" - " - y\n" - " command:\n" - " backup: {}\n" - " command-role:\n" - " main: {}\n" - "\n" - " backup-standby:\n" - " section: global\n" - " type: boolean\n" - " beta: true\n" - " default: false\n" - " depend:\n" - " option: online\n" - " default: true\n" - " list:\n" - " - true\n" - " command:\n" - " backup: {}\n" - " command-role:\n" - " async: {}\n" - " main: {}\n" - " deprecate:\n" - " backup-standby-old: {}\n" - "\n" - " backup-copy:\n" - " section: global\n" - " type: boolean\n" - " default: false\n" - " depend: backup-standby\n" - " command:\n" - " +role: any\n" - " -command: archive-get\n" - " backup: {}\n" - " command-role:\n" - " main: {}\n" - "\n" - " pg-path:\n" - " section: stanza\n" - " group: pg\n" - " type: hash\n" - " deprecate:\n" - " pg-path: {}\n" - " pg?-path-indexed: {}\n" - " db-path: {}\n" - " db?-path: {}\n" - "\n" - " pg-host:\n" - " section: stanza\n" - " group: pg\n" - " type: string\n" - " command:\n" - " archive-get:\n" - " default: host1\n" - " backup: {}\n" - " deprecate:\n" - " pg?-host: {}\n"); - - TEST_RESULT_VOID(bldCfgRender(storageTest, bldCfgParse(storageTest), false), "parse and render"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("bldCfgRenderVar128Size()"); - - TEST_RESULT_UINT(bldCfgRenderVar128Size(10000), 2, "check size"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("check config.auto.h"); - - TEST_STORAGE_GET( - storageTest, - "src/config/config.auto.h", - COMMENT_BLOCK_BEGIN "\n" - "Command and Option Configuration\n" - "\n" - "Automatically generated by 'build-code config' -- do not modify directly.\n" - COMMENT_BLOCK_END "\n" - "#ifndef CONFIG_CONFIG_AUTO_H\n" - "#define CONFIG_CONFIG_AUTO_H\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Command constants\n" - COMMENT_BLOCK_END "\n" - "#define CFGCMD_ARCHIVE_GET \"archive-get\"\n" - "#define CFGCMD_BACKUP \"backup\"\n" - "#define CFGCMD_HELP \"help\"\n" - "#define CFGCMD_VERSION \"version\"\n" - "\n" - "#define CFG_COMMAND_TOTAL 4\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Option group constants\n" - COMMENT_BLOCK_END "\n" - "#define CFG_OPTION_GROUP_TOTAL 2\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Option constants\n" - COMMENT_BLOCK_END "\n" - "#define CFGOPT_BACKUP_COPY \"backup-copy\"\n" - "#define CFGOPT_BACKUP_STANDBY \"backup-standby\"\n" - "#define CFGOPT_BOOL_LIKE \"bool-like\"\n" - "#define CFGOPT_BUFFER_SIZE \"buffer-size\"\n" - "#define CFGOPT_CMD \"cmd\"\n" - "#define CFGOPT_COMPRESS_LEVEL \"compress-level\"\n" - "#define CFGOPT_COMPRESS_LEVEL_NETWORK \"compress-level-network\"\n" - "#define CFGOPT_COMPRESS_NETWORK \"compress-network\"\n" - "#define CFGOPT_COMPRESS_TYPE \"compress-type\"\n" - "#define CFGOPT_CONFIG \"config\"\n" - "#define CFGOPT_CONFIG_INCLUDE \"config-include\"\n" - "#define CFGOPT_LOG_LEVEL_CONSOLE \"log-level-console\"\n" - "#define CFGOPT_LOG_LEVEL_FILE \"log-level-file\"\n" - "#define CFGOPT_ONLINE \"online\"\n" - "#define CFGOPT_STANZA \"stanza\"\n" - "#define CFGOPT_TIMEOUT \"timeout\"\n" - "\n" - "#define CFG_OPTION_TOTAL 18\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Option value constants\n" - COMMENT_BLOCK_END "\n" - "#define CFGOPTVAL_BOOL_LIKE_N 0\n" - "#define CFGOPTVAL_BOOL_LIKE_N_STRID STRID5S(\"n\", 0, 0xe2)\n" - "#define CFGOPTVAL_BOOL_LIKE_N_Z \"n\"\n" - "#define CFGOPTVAL_BOOL_LIKE_CHECK 1\n" - "#define CFGOPTVAL_BOOL_LIKE_CHECK_STRID STRID5S(\"check\", 1, 0xb195034)\n" - "#define CFGOPTVAL_BOOL_LIKE_CHECK_Z \"check\"\n" - "#define CFGOPTVAL_BOOL_LIKE_Y 2\n" - "#define CFGOPTVAL_BOOL_LIKE_Y_STRID STRID5S(\"y\", 2, 0x196)\n" - "#define CFGOPTVAL_BOOL_LIKE_Y_Z \"y\"\n" - "\n" - "#define CFGOPTVAL_COMPRESS_NETWORK_GZ STRID5(\"gz\", 0x3470)\n" - "#define CFGOPTVAL_COMPRESS_NETWORK_GZ_Z \"gz\"\n" - "\n" - "#define CFGOPTVAL_ARCHIVE_GET_COMPRESS_NETWORK_BZ2 0\n" - "#define CFGOPTVAL_ARCHIVE_GET_COMPRESS_NETWORK_BZ2_STRID STRID5S(\"bz2\", 0, 0x73422)\n" - "#define CFGOPTVAL_ARCHIVE_GET_COMPRESS_NETWORK_BZ2_Z \"bz2\"\n" - "\n" - "#define CFGOPTVAL_LOG_LEVEL_CONSOLE_DEBUG1 STRID6(\"debug1\", 0x7475421441)\n" - "#define CFGOPTVAL_LOG_LEVEL_CONSOLE_DEBUG1_Z \"debug1\"\n" - "#define CFGOPTVAL_LOG_LEVEL_CONSOLE_ERROR STRID5(\"error\", 0x127ca450)\n" - "#define CFGOPTVAL_LOG_LEVEL_CONSOLE_ERROR_Z \"error\"\n" - "#define CFGOPTVAL_LOG_LEVEL_CONSOLE_OFF STRID5(\"off\", 0x18cf0)\n" - "#define CFGOPTVAL_LOG_LEVEL_CONSOLE_OFF_Z \"off\"\n" - "#define CFGOPTVAL_LOG_LEVEL_CONSOLE_WARN STRID5(\"warn\", 0x748370)\n" - "#define CFGOPTVAL_LOG_LEVEL_CONSOLE_WARN_Z \"warn\"\n" - "\n" - "#define CFGOPTVAL_LOG_LEVEL_FILE_DEBUG1 STRID6(\"debug1\", 0x7475421441)\n" - "#define CFGOPTVAL_LOG_LEVEL_FILE_DEBUG1_Z \"debug1\"\n" - "#define CFGOPTVAL_LOG_LEVEL_FILE_ERROR STRID5(\"error\", 0x127ca450)\n" - "#define CFGOPTVAL_LOG_LEVEL_FILE_ERROR_Z \"error\"\n" - "#define CFGOPTVAL_LOG_LEVEL_FILE_OFF STRID5(\"off\", 0x18cf0)\n" - "#define CFGOPTVAL_LOG_LEVEL_FILE_OFF_Z \"off\"\n" - "#define CFGOPTVAL_LOG_LEVEL_FILE_WARN STRID5(\"warn\", 0x748370)\n" - "#define CFGOPTVAL_LOG_LEVEL_FILE_WARN_Z \"warn\"\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Command enum\n" - COMMENT_BLOCK_END "\n" - "typedef enum\n" - "{\n" - " cfgCmdArchiveGet,\n" - " cfgCmdBackup,\n" - " cfgCmdHelp,\n" - " cfgCmdVersion,\n" - "} ConfigCommand;\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Option group enum\n" - COMMENT_BLOCK_END "\n" - "typedef enum\n" - "{\n" - " cfgOptGrpPg,\n" - " cfgOptGrpRepo,\n" - "} ConfigOptionGroup;\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Option enum\n" - COMMENT_BLOCK_END "\n" - "typedef enum\n" - "{\n" - " cfgOptBackupCopy,\n" - " cfgOptBackupStandby,\n" - " cfgOptBoolLike,\n" - " cfgOptBufferSize,\n" - " cfgOptCmd,\n" - " cfgOptCompressLevel,\n" - " cfgOptCompressLevelNetwork,\n" - " cfgOptCompressNetwork,\n" - " cfgOptCompressType,\n" - " cfgOptConfig,\n" - " cfgOptConfigInclude,\n" - " cfgOptLogLevelConsole,\n" - " cfgOptLogLevelFile,\n" - " cfgOptOnline,\n" - " cfgOptPgHost,\n" - " cfgOptPgPath,\n" - " cfgOptStanza,\n" - " cfgOptTimeout,\n" - "} ConfigOption;\n" - "\n" - "#endif\n"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("check parse.auto.c.inc"); - - TEST_STORAGE_GET( - storageTest, - "src/config/parse.auto.c.inc", - COMMENT_BLOCK_BEGIN "\n" - "Config Parse Rules\n" - "\n" - "Automatically generated by 'build-code config' -- do not modify directly.\n" - COMMENT_BLOCK_END "\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Rule Strings\n" - COMMENT_BLOCK_END "\n" - "#define PARSE_RULE_VAL_STR(value) PARSE_RULE_U32_1(parseRuleValStr##value)\n" - "#define PARSE_RULE_VAL_STRID(value) PARSE_RULE_VAL_STR(QT_##value##_QT)\n" - "\n" - "static const StringPubConst parseRuleValueStr[] =\n" - "{\n" - " PARSE_RULE_STRPUB(\"-1\"),\n" - " PARSE_RULE_STRPUB(\"/include\"),\n" - " PARSE_RULE_STRPUB(\"1\"),\n" - " PARSE_RULE_STRPUB(\"100ms\"),\n" - " PARSE_RULE_STRPUB(\"10s\"),\n" - " PARSE_RULE_STRPUB(\"16KiB\"),\n" - " PARSE_RULE_STRPUB(\"30m\"),\n" - " PARSE_RULE_STRPUB(\"32KiB\"),\n" - " PARSE_RULE_STRPUB(\"6\"),\n" - " PARSE_RULE_STRPUB(\"7w\"),\n" - " PARSE_RULE_STRPUB(\"8KiB\"),\n" - " PARSE_RULE_STRPUB(\"9\"),\n" - " PARSE_RULE_STRPUB(\"bz2\"),\n" - " PARSE_RULE_STRPUB(\"check\"),\n" - " PARSE_RULE_STRPUB(\"debug1\"),\n" - " PARSE_RULE_STRPUB(\"error\"),\n" - " PARSE_RULE_STRPUB(\"gz\"),\n" - " PARSE_RULE_STRPUB(\"host1\"),\n" - " PARSE_RULE_STRPUB(\"info\"),\n" - " PARSE_RULE_STRPUB(\"n\"),\n" - " PARSE_RULE_STRPUB(\"none\"),\n" - " PARSE_RULE_STRPUB(\"off\"),\n" - " PARSE_RULE_STRPUB(\"warn\"),\n" - " PARSE_RULE_STRPUB(\"y\"),\n" - " PARSE_RULE_STRPUB(CFGOPTDEF_CONFIG_PATH \"/\" PROJECT_CONFIG_FILE),\n" - " PARSE_RULE_STRPUB(CFGOPTDEF_CONFIG_PATH \"/.-\" PROJECT_CONFIG_FILE),\n" - "};\n" - "\n" - "typedef enum\n" - "{\n" - " parseRuleValStrQT_DS_1_QT,\n" - " parseRuleValStrQT_FS_include_QT,\n" - " parseRuleValStrQT_1_QT,\n" - " parseRuleValStrQT_100ms_QT,\n" - " parseRuleValStrQT_10s_QT,\n" - " parseRuleValStrQT_16KiB_QT,\n" - " parseRuleValStrQT_30m_QT,\n" - " parseRuleValStrQT_32KiB_QT,\n" - " parseRuleValStrQT_6_QT,\n" - " parseRuleValStrQT_7w_QT,\n" - " parseRuleValStrQT_8KiB_QT,\n" - " parseRuleValStrQT_9_QT,\n" - " parseRuleValStrQT_bz2_QT,\n" - " parseRuleValStrQT_check_QT,\n" - " parseRuleValStrQT_debug1_QT,\n" - " parseRuleValStrQT_error_QT,\n" - " parseRuleValStrQT_gz_QT,\n" - " parseRuleValStrQT_host1_QT,\n" - " parseRuleValStrQT_info_QT,\n" - " parseRuleValStrQT_n_QT,\n" - " parseRuleValStrQT_none_QT,\n" - " parseRuleValStrQT_off_QT,\n" - " parseRuleValStrQT_warn_QT,\n" - " parseRuleValStrQT_y_QT,\n" - " parseRuleValStrCFGOPTDEF_CONFIG_PATH_SP_QT_FS_QT_SP_PROJECT_CONFIG_FILE,\n" - " parseRuleValStrCFGOPTDEF_CONFIG_PATH_SP_QT_FS_DT_DS_QT_SP_PROJECT_CONFIG_FILE,\n" - "} ParseRuleValueStr;\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Rule Ints\n" - COMMENT_BLOCK_END "\n" - "#define PARSE_RULE_VAL_INT(value) PARSE_RULE_U32_1(parseRuleValInt##value)\n" - "\n" - "static const int parseRuleValueInt[] =\n" - "{\n" - " -1,\n" - " 1,\n" - " 6,\n" - " 9,\n" - "};\n" - "\n" - "static const uint8_t parseRuleValueIntStrMap[] =\n" - "{\n" - " parseRuleValStrQT_DS_1_QT,\n" - " parseRuleValStrQT_1_QT,\n" - " parseRuleValStrQT_6_QT,\n" - " parseRuleValStrQT_9_QT,\n" - "};\n" - "\n" - "typedef enum\n" - "{\n" - " parseRuleValIntDS_1,\n" - " parseRuleValInt1,\n" - " parseRuleValInt6,\n" - " parseRuleValInt9,\n" - "} ParseRuleValueInt;\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Rule Sizes\n" - COMMENT_BLOCK_END "\n" - "#define PARSE_RULE_VAL_SIZE(value) PARSE_RULE_U32_1(parseRuleValSize##value)\n" - "\n" - "static const int64_t parseRuleValueSize[] =\n" - "{\n" - " 8192,\n" - " 16384,\n" - " 32768,\n" - "};\n" - "\n" - "static const uint8_t parseRuleValueSizeStrMap[] =\n" - "{\n" - " parseRuleValStrQT_8KiB_QT,\n" - " parseRuleValStrQT_16KiB_QT,\n" - " parseRuleValStrQT_32KiB_QT,\n" - "};\n" - "\n" - "typedef enum\n" - "{\n" - " parseRuleValSize8KiB,\n" - " parseRuleValSize16KiB,\n" - " parseRuleValSize32KiB,\n" - "} ParseRuleValueSize;\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Rule Times\n" - COMMENT_BLOCK_END "\n" - "#define PARSE_RULE_VAL_TIME(value) PARSE_RULE_U32_1(parseRuleValTime##value)\n" - "\n" - "static const unsigned int parseRuleValueTime[] =\n" - "{\n" - " 100,\n" - " 10000,\n" - " 1800000,\n" - " 4233600000,\n" - "};\n" - "\n" - "static const uint8_t parseRuleValueTimeStrMap[] =\n" - "{\n" - " parseRuleValStrQT_100ms_QT,\n" - " parseRuleValStrQT_10s_QT,\n" - " parseRuleValStrQT_30m_QT,\n" - " parseRuleValStrQT_7w_QT,\n" - "};\n" - "\n" - "typedef enum\n" - "{\n" - " parseRuleValTime100ms,\n" - " parseRuleValTime10s,\n" - " parseRuleValTime30m,\n" - " parseRuleValTime7w,\n" - "} ParseRuleValueTime;\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Dynamic default values\n" - COMMENT_BLOCK_END "\n" - "#define PARSE_RULE_DEFAULT_DYNAMIC(value) PARSE_RULE_U32_1(parseRuleDefaultDynamic##value)\n" - "\n" - "typedef enum\n" - "{\n" - " parseRuleDefaultDynamicBin,\n" - "} ParseRuleDefaultDynamic;\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Command parse data\n" - COMMENT_BLOCK_END "\n" - "#define PARSE_RULE_VAL_CMD(value) PARSE_RULE_U32_1(cfgCmd##value)\n" - "\n" - "static const ParseRuleCommand parseRuleCommand[CFG_COMMAND_TOTAL] =\n" - "{\n" - " PARSE_RULE_COMMAND\n" - " (\n" - " PARSE_RULE_COMMAND_NAME(\"archive-get\"),\n" - " PARSE_RULE_COMMAND_LOCK_TYPE(Archive),\n" - " PARSE_RULE_COMMAND_LOG_LEVEL_DEFAULT(Debug),\n" - " PARSE_RULE_COMMAND_PARAMETER_ALLOWED(true),\n" - "\n" - " PARSE_RULE_COMMAND_ROLE_VALID_LIST\n" - " (\n" - " PARSE_RULE_COMMAND_ROLE(Async)\n" - " PARSE_RULE_COMMAND_ROLE(Local)\n" - " PARSE_RULE_COMMAND_ROLE(Main)\n" - " PARSE_RULE_COMMAND_ROLE(Remote)\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_COMMAND\n" - " (\n" - " PARSE_RULE_COMMAND_NAME(\"backup\"),\n" - " PARSE_RULE_COMMAND_INTERNAL(true)\n" - " PARSE_RULE_COMMAND_LOCK_REQUIRED(true),\n" - " PARSE_RULE_COMMAND_LOCK_REMOTE_REQUIRED(true),\n" - " PARSE_RULE_COMMAND_LOCK_TYPE(Backup),\n" - " PARSE_RULE_COMMAND_LOG_FILE(true),\n" - " PARSE_RULE_COMMAND_LOG_LEVEL_DEFAULT(Info),\n" - "\n" - " PARSE_RULE_COMMAND_ROLE_VALID_LIST\n" - " (\n" - " PARSE_RULE_COMMAND_ROLE(Local)\n" - " PARSE_RULE_COMMAND_ROLE(Main)\n" - " PARSE_RULE_COMMAND_ROLE(Remote)\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_COMMAND\n" - " (\n" - " PARSE_RULE_COMMAND_NAME(\"help\"),\n" - " PARSE_RULE_COMMAND_LOCK_TYPE(None),\n" - " PARSE_RULE_COMMAND_LOG_FILE(true),\n" - " PARSE_RULE_COMMAND_LOG_LEVEL_DEFAULT(Info),\n" - "\n" - " PARSE_RULE_COMMAND_ROLE_VALID_LIST\n" - " (\n" - " PARSE_RULE_COMMAND_ROLE(Main)\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_COMMAND\n" - " (\n" - " PARSE_RULE_COMMAND_NAME(\"version\"),\n" - " PARSE_RULE_COMMAND_LOCK_TYPE(None),\n" - " PARSE_RULE_COMMAND_LOG_FILE(true),\n" - " PARSE_RULE_COMMAND_LOG_LEVEL_DEFAULT(Info),\n" - "\n" - " PARSE_RULE_COMMAND_ROLE_VALID_LIST\n" - " (\n" - " PARSE_RULE_COMMAND_ROLE(Main)\n" - " ),\n" - " ),\n" - "};\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Option group parse data\n" - COMMENT_BLOCK_END "\n" - "static const ParseRuleOptionGroup parseRuleOptionGroup[CFG_OPTION_GROUP_TOTAL] =\n" - "{\n" - " PARSE_RULE_OPTION_GROUP\n" - " (\n" - " PARSE_RULE_OPTION_GROUP_NAME(\"pg\"),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_OPTION_GROUP\n" - " (\n" - " PARSE_RULE_OPTION_GROUP_NAME(\"repo\"),\n" - " ),\n" - "};\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Option parse data\n" - COMMENT_BLOCK_END "\n" - "#define PARSE_RULE_VAL_OPT(value) PARSE_RULE_U32_1(cfgOpt##value)\n" - "\n" - "static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =\n" - "{\n" - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"backup-copy\"),\n" - " PARSE_RULE_OPTION_TYPE(Boolean),\n" - " PARSE_RULE_OPTION_NEGATE(true),\n" - " PARSE_RULE_OPTION_RESET(true),\n" - " PARSE_RULE_OPTION_REQUIRED(true),\n" - " PARSE_RULE_OPTION_SECTION(Global),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL\n" - " (\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_OPTIONAL_DEPEND\n" - " (\n" - " PARSE_RULE_OPTIONAL_DEPEND_DEFAULT(PARSE_RULE_VAL_BOOL_TRUE),\n" - " PARSE_RULE_VAL_OPT(Online),\n" - " PARSE_RULE_VAL_BOOL_TRUE,\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_VAL_BOOL_FALSE,\n" - " ),\n" - " ),\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"backup-standby\"),\n" - " PARSE_RULE_OPTION_TYPE(Boolean),\n" - " PARSE_RULE_OPTION_BETA(true),\n" - " PARSE_RULE_OPTION_NEGATE(true),\n" - " PARSE_RULE_OPTION_RESET(true),\n" - " PARSE_RULE_OPTION_REQUIRED(true),\n" - " PARSE_RULE_OPTION_SECTION(Global),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL\n" - " (\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_OPTIONAL_DEPEND\n" - " (\n" - " PARSE_RULE_OPTIONAL_DEPEND_DEFAULT(PARSE_RULE_VAL_BOOL_TRUE),\n" - " PARSE_RULE_VAL_OPT(Online),\n" - " PARSE_RULE_VAL_BOOL_TRUE,\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_VAL_BOOL_FALSE,\n" - " ),\n" - " ),\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"bool-like\"),\n" - " PARSE_RULE_OPTION_TYPE(StringId),\n" - " PARSE_RULE_OPTION_SEQUENCE(true),\n" - " PARSE_RULE_OPTION_BOOL_LIKE(true),\n" - " PARSE_RULE_OPTION_NEGATE(true),\n" - " PARSE_RULE_OPTION_RESET(true),\n" - " PARSE_RULE_OPTION_REQUIRED(true),\n" - " PARSE_RULE_OPTION_SECTION(Global),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL\n" - " (\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_OPTIONAL_ALLOW_LIST\n" - " (\n" - " PARSE_RULE_VAL_STRID(n),\n" - " PARSE_RULE_VAL_STRID(check),\n" - " PARSE_RULE_VAL_STRID(y),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_VAL_STRID(n),\n" - " ),\n" - " ),\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"buffer-size\"),\n" - " PARSE_RULE_OPTION_TYPE(Size),\n" - " PARSE_RULE_OPTION_RESET(true),\n" - " PARSE_RULE_OPTION_REQUIRED(true),\n" - " PARSE_RULE_OPTION_SECTION(Global),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL\n" - " (\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_FILTER_CMD\n" - " (\n" - " PARSE_RULE_VAL_CMD(ArchiveGet),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_ALLOW_LIST\n" - " (\n" - " PARSE_RULE_VAL_SIZE(32KiB),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_VAL_SIZE(32KiB),\n" - " ),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_OPTIONAL_ALLOW_LIST\n" - " (\n" - " PARSE_RULE_VAL_SIZE(8KiB),\n" - " PARSE_RULE_VAL_SIZE(16KiB),\n" - "#ifndef HAVE_LARGE_MEM\n" - " PARSE_RULE_BOOL_FALSE,\n" - "#endif\n" - " ),\n" - " ),\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"cmd\"),\n" - " PARSE_RULE_OPTION_TYPE(String),\n" - " PARSE_RULE_OPTION_DEFAULT_TYPE(Dynamic),\n" - " PARSE_RULE_OPTION_RESET(true),\n" - " PARSE_RULE_OPTION_REQUIRED(true),\n" - " PARSE_RULE_OPTION_SECTION(Global),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL\n" - " (\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_DEFAULT_DYNAMIC(Bin),\n" - " ),\n" - " ),\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"compress-level\"),\n" - " PARSE_RULE_OPTION_TYPE(Integer),\n" - " PARSE_RULE_OPTION_RESET(true),\n" - " PARSE_RULE_OPTION_REQUIRED(false),\n" - " PARSE_RULE_OPTION_SECTION(Global),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL\n" - " (\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_FILTER_CMD\n" - " (\n" - " PARSE_RULE_VAL_CMD(ArchiveGet),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEPEND\n" - " (\n" - " PARSE_RULE_VAL_OPT(Config),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_ALLOW_RANGE\n" - " (\n" - " PARSE_RULE_OPTIONAL_ALLOW_RANGE_MAP\n" - " (\n" - " PARSE_RULE_VAL_STRID(bz2),\n" - " PARSE_RULE_VAL_INT(1),\n" - " PARSE_RULE_VAL_INT(9),\n" - " PARSE_RULE_VAL_STRID(gz),\n" - " PARSE_RULE_VAL_INT(DS_1),\n" - " PARSE_RULE_VAL_INT(9),\n" - " ),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_OPTIONAL_DEFAULT_MAP\n" - " (\n" - " PARSE_RULE_VAL_STRID(bz2),\n" - " PARSE_RULE_VAL_INT(9),\n" - " PARSE_RULE_VAL_STRID(gz),\n" - " PARSE_RULE_VAL_INT(6),\n" - " ),\n" - " ),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_OPTIONAL_DEPEND\n" - " (\n" - " PARSE_RULE_OPTIONAL_DEPEND_DEFAULT(PARSE_RULE_VAL_INT(0)),\n" - " PARSE_RULE_VAL_OPT(CompressType),\n" - " PARSE_RULE_VAL_STRID(bz2),\n" - " PARSE_RULE_VAL_STRID(gz),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_ALLOW_RANGE\n" - " (\n" - " PARSE_RULE_OPTIONAL_ALLOW_RANGE_MAP\n" - " (\n" - " PARSE_RULE_VAL_STRID(bz2),\n" - " PARSE_RULE_VAL_INT(1),\n" - " PARSE_RULE_VAL_INT(9),\n" - " PARSE_RULE_VAL_STRID(gz),\n" - " PARSE_RULE_VAL_INT(DS_1),\n" - " PARSE_RULE_VAL_INT(9),\n" - " ),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_OPTIONAL_DEFAULT_MAP\n" - " (\n" - " PARSE_RULE_VAL_STRID(bz2),\n" - " PARSE_RULE_VAL_INT(9),\n" - " PARSE_RULE_VAL_STRID(gz),\n" - " PARSE_RULE_VAL_INT(6),\n" - " ),\n" - " ),\n" - " ),\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"compress-level-network\"),\n" - " PARSE_RULE_OPTION_TYPE(Integer),\n" - " PARSE_RULE_OPTION_INTERNAL(true)\n" - " PARSE_RULE_OPTION_RESET(true),\n" - " PARSE_RULE_OPTION_REQUIRED(false),\n" - " PARSE_RULE_OPTION_SECTION(Global),\n" - " PARSE_RULE_OPTION_SECURE(true),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL\n" - " (\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_FILTER_CMD\n" - " (\n" - " PARSE_RULE_VAL_CMD(ArchiveGet),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEPEND\n" - " (\n" - " PARSE_RULE_VAL_OPT(Config),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_ALLOW_RANGE\n" - " (\n" - " PARSE_RULE_OPTIONAL_ALLOW_RANGE_MAP\n" - " (\n" - " PARSE_RULE_VAL_STRID(bz2),\n" - " PARSE_RULE_VAL_INT(1),\n" - " PARSE_RULE_VAL_INT(9),\n" - " PARSE_RULE_VAL_STRID(gz),\n" - " PARSE_RULE_VAL_INT(DS_1),\n" - " PARSE_RULE_VAL_INT(9),\n" - " ),\n" - " ),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_OPTIONAL_DEPEND\n" - " (\n" - " PARSE_RULE_VAL_OPT(CompressType),\n" - " PARSE_RULE_VAL_STRID(none),\n" - " PARSE_RULE_VAL_STRID(gz),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_ALLOW_RANGE\n" - " (\n" - " PARSE_RULE_OPTIONAL_ALLOW_RANGE_MAP\n" - " (\n" - " PARSE_RULE_VAL_STRID(bz2),\n" - " PARSE_RULE_VAL_INT(1),\n" - " PARSE_RULE_VAL_INT(9),\n" - " PARSE_RULE_VAL_STRID(gz),\n" - " PARSE_RULE_VAL_INT(DS_1),\n" - " PARSE_RULE_VAL_INT(9),\n" - " ),\n" - " ),\n" - " ),\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"compress-network\"),\n" - " PARSE_RULE_OPTION_TYPE(StringId),\n" - " PARSE_RULE_OPTION_RESET(true),\n" - " PARSE_RULE_OPTION_REQUIRED(true),\n" - " PARSE_RULE_OPTION_SECTION(Global),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL\n" - " (\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_FILTER_CMD\n" - " (\n" - " PARSE_RULE_VAL_CMD(ArchiveGet),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_SEQUENCE(),\n" - "\n" - " PARSE_RULE_OPTIONAL_ALLOW_LIST\n" - " (\n" - " PARSE_RULE_VAL_STRID(bz2),\n" - " ),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_FILTER_CMD\n" - " (\n" - " PARSE_RULE_VAL_CMD(Backup),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_ALLOW_LIST\n" - " (\n" - " PARSE_RULE_VAL_STRID(gz),\n" - " ),\n" - " ),\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"compress-type\"),\n" - " PARSE_RULE_OPTION_TYPE(String),\n" - " PARSE_RULE_OPTION_SEQUENCE(true),\n" - " PARSE_RULE_OPTION_RESET(true),\n" - " PARSE_RULE_OPTION_REQUIRED(true),\n" - " PARSE_RULE_OPTION_SECTION(Global),\n" - "\n" - " PARSE_RULE_OPTIONAL\n" - " (\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_FILTER_CMD\n" - " (\n" - " PARSE_RULE_VAL_CMD(ArchiveGet),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEPEND\n" - " (\n" - " PARSE_RULE_VAL_OPT(Config),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_ALLOW_LIST\n" - " (\n" - " PARSE_RULE_VAL_STR(QT_none_QT),\n" - " PARSE_RULE_VAL_STR(QT_bz2_QT),\n" - " PARSE_RULE_VAL_STR(QT_gz_QT),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_VAL_STR(QT_gz_QT),\n" - " PARSE_RULE_VAL_SEQ(2),\n" - " ),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_OPTIONAL_ALLOW_LIST\n" - " (\n" - " PARSE_RULE_VAL_STR(QT_none_QT),\n" - " PARSE_RULE_VAL_STR(QT_bz2_QT),\n" - " PARSE_RULE_VAL_STR(QT_gz_QT),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_VAL_STR(QT_gz_QT),\n" - " PARSE_RULE_VAL_SEQ(2),\n" - " ),\n" - " ),\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"config\"),\n" - " PARSE_RULE_OPTION_TYPE(String),\n" - " PARSE_RULE_OPTION_NEGATE(true),\n" - " PARSE_RULE_OPTION_REQUIRED(true),\n" - " PARSE_RULE_OPTION_SECTION(CommandLine),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL\n" - " (\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_FILTER_CMD\n" - " (\n" - " PARSE_RULE_VAL_CMD(ArchiveGet),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_VAL_STR(CFGOPTDEF_CONFIG_PATH_SP_QT_FS_DT_DS_QT_SP_PROJECT_CONFIG_FILE),\n" - " ),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_VAL_STR(CFGOPTDEF_CONFIG_PATH_SP_QT_FS_QT_SP_PROJECT_CONFIG_FILE),\n" - " ),\n" - " ),\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"config-include\"),\n" - " PARSE_RULE_OPTION_TYPE(Path),\n" - " PARSE_RULE_OPTION_RESET(true),\n" - " PARSE_RULE_OPTION_REQUIRED(true),\n" - " PARSE_RULE_OPTION_SECTION(Global),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL\n" - " (\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_VAL_STR(QT_FS_include_QT),\n" - " ),\n" - " ),\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"log-level-console\"),\n" - " PARSE_RULE_OPTION_TYPE(StringId),\n" - " PARSE_RULE_OPTION_RESET(true),\n" - " PARSE_RULE_OPTION_REQUIRED(true),\n" - " PARSE_RULE_OPTION_SECTION(Global),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL\n" - " (\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_OPTIONAL_ALLOW_LIST\n" - " (\n" - " PARSE_RULE_VAL_STRID(off),\n" - " PARSE_RULE_VAL_STRID(error),\n" - " PARSE_RULE_VAL_STRID(warn),\n" - " PARSE_RULE_VAL_STRID(debug1),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_VAL_STRID(warn),\n" - " ),\n" - " ),\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"log-level-file\"),\n" - " PARSE_RULE_OPTION_TYPE(StringId),\n" - " PARSE_RULE_OPTION_RESET(true),\n" - " PARSE_RULE_OPTION_REQUIRED(true),\n" - " PARSE_RULE_OPTION_SECTION(Global),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_INTERNAL_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND_INTERNAL(Backup, true),\n" - " )\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " PARSE_RULE_OPTION_COMMAND(Help)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL\n" - " (\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_FILTER_CMD\n" - " (\n" - " PARSE_RULE_VAL_CMD(Backup),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEPEND\n" - " (\n" - " PARSE_RULE_VAL_OPT(LogLevelConsole),\n" - " PARSE_RULE_VAL_STRID(warn),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_ALLOW_LIST\n" - " (\n" - " PARSE_RULE_VAL_STRID(off),\n" - " PARSE_RULE_VAL_STRID(warn),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_VAL_STRID(warn),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_NOT_REQUIRED(),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_OPTIONAL_ALLOW_LIST\n" - " (\n" - " PARSE_RULE_VAL_STRID(off),\n" - " PARSE_RULE_VAL_STRID(error),\n" - " PARSE_RULE_VAL_STRID(warn),\n" - " PARSE_RULE_VAL_STRID(debug1),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_VAL_STRID(info),\n" - " ),\n" - " ),\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"online\"),\n" - " PARSE_RULE_OPTION_TYPE(Boolean),\n" - " PARSE_RULE_OPTION_REQUIRED(true),\n" - " PARSE_RULE_OPTION_SECTION(CommandLine),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL\n" - " (\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_FILTER_CMD\n" - " (\n" - " PARSE_RULE_VAL_CMD(Backup),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_VAL_BOOL_FALSE,\n" - " ),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_VAL_BOOL_TRUE,\n" - " ),\n" - " ),\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"pg-host\"),\n" - " PARSE_RULE_OPTION_TYPE(String),\n" - " PARSE_RULE_OPTION_RESET(true),\n" - " PARSE_RULE_OPTION_REQUIRED(true),\n" - " PARSE_RULE_OPTION_SECTION(Stanza),\n" - " PARSE_RULE_OPTION_GROUP_ID(Pg),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL\n" - " (\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_FILTER_CMD\n" - " (\n" - " PARSE_RULE_VAL_CMD(ArchiveGet),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_VAL_STR(QT_host1_QT),\n" - " ),\n" - " ),\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"pg-path\"),\n" - " PARSE_RULE_OPTION_TYPE(Hash),\n" - " PARSE_RULE_OPTION_RESET(true),\n" - " PARSE_RULE_OPTION_REQUIRED(true),\n" - " PARSE_RULE_OPTION_SECTION(Stanza),\n" - " PARSE_RULE_OPTION_MULTI(true),\n" - " PARSE_RULE_OPTION_GROUP_ID(Pg),\n" - " PARSE_RULE_OPTION_DEPRECATE_MATCH(true),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"stanza\"),\n" - " PARSE_RULE_OPTION_TYPE(List),\n" - " PARSE_RULE_OPTION_REQUIRED(true),\n" - " PARSE_RULE_OPTION_SECTION(CommandLine),\n" - " PARSE_RULE_OPTION_MULTI(true),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - " ),\n" - COMMENT_SEPARATOR "\n" - " PARSE_RULE_OPTION\n" - " (\n" - " PARSE_RULE_OPTION_NAME(\"timeout\"),\n" - " PARSE_RULE_OPTION_TYPE(Time),\n" - " PARSE_RULE_OPTION_REQUIRED(true),\n" - " PARSE_RULE_OPTION_SECTION(CommandLine),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST\n" - " (\n" - " PARSE_RULE_OPTION_COMMAND(ArchiveGet)\n" - " PARSE_RULE_OPTION_COMMAND(Backup)\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL\n" - " (\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_FILTER_CMD\n" - " (\n" - " PARSE_RULE_VAL_CMD(ArchiveGet),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_ALLOW_RANGE\n" - " (\n" - " PARSE_RULE_VAL_TIME(100ms),\n" - " PARSE_RULE_VAL_TIME(7w),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_VAL_TIME(30m),\n" - " ),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_FILTER_CMD\n" - " (\n" - " PARSE_RULE_VAL_CMD(Backup),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_ALLOW_RANGE\n" - " (\n" - " PARSE_RULE_VAL_TIME(100ms),\n" - " PARSE_RULE_VAL_TIME(7w),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_VAL_TIME(100ms),\n" - " ),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_GROUP\n" - " (\n" - " PARSE_RULE_OPTIONAL_ALLOW_RANGE\n" - " (\n" - " PARSE_RULE_VAL_TIME(100ms),\n" - " PARSE_RULE_VAL_TIME(7w),\n" - " ),\n" - "\n" - " PARSE_RULE_OPTIONAL_DEFAULT\n" - " (\n" - " PARSE_RULE_VAL_TIME(10s),\n" - " ),\n" - " ),\n" - " ),\n" - " ),\n" - "};\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Option deprecations\n" - COMMENT_BLOCK_END "\n" - "#define CFG_OPTION_DEPRECATE_TOTAL 7\n" - "\n" - "static const ParseRuleOptionDeprecate parseRuleOptionDeprecate[CFG_OPTION_DEPRECATE_TOTAL] =\n" - "{\n" - " {\n" - " .name = \"backup-standby-old\",\n" - " .id = cfgOptBackupStandby,\n" - " .unindexed = true,\n" - " },\n" - COMMENT_SEPARATOR "\n" - " {\n" - " .name = \"compress\",\n" - " .id = cfgOptCompressType,\n" - " .unindexed = true,\n" - " },\n" - COMMENT_SEPARATOR "\n" - " {\n" - " .name = \"db-path\",\n" - " .id = cfgOptPgPath,\n" - " .indexed = true,\n" - " .unindexed = true,\n" - " },\n" - COMMENT_SEPARATOR "\n" - " {\n" - " .name = \"online-old\",\n" - " .id = cfgOptOnline,\n" - " .unindexed = true,\n" - " },\n" - COMMENT_SEPARATOR "\n" - " {\n" - " .name = \"pg-host\",\n" - " .id = cfgOptPgHost,\n" - " .indexed = true,\n" - " },\n" - COMMENT_SEPARATOR "\n" - " {\n" - " .name = \"pg-path\",\n" - " .id = cfgOptPgPath,\n" - " .unindexed = true,\n" - " },\n" - COMMENT_SEPARATOR "\n" - " {\n" - " .name = \"pg-path-indexed\",\n" - " .id = cfgOptPgPath,\n" - " .indexed = true,\n" - " },\n" - "};\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Order for option parse resolution\n" - COMMENT_BLOCK_END "\n" - "static const uint8_t optionResolveOrder[] =\n" - "{\n" - " cfgOptStanza,\n" - " cfgOptBoolLike,\n" - " cfgOptBufferSize,\n" - " cfgOptCmd,\n" - " cfgOptCompressNetwork,\n" - " cfgOptConfig,\n" - " cfgOptConfigInclude,\n" - " cfgOptLogLevelConsole,\n" - " cfgOptLogLevelFile,\n" - " cfgOptOnline,\n" - " cfgOptPgHost,\n" - " cfgOptPgPath,\n" - " cfgOptTimeout,\n" - " cfgOptBackupCopy,\n" - " cfgOptBackupStandby,\n" - " cfgOptCompressType,\n" - " cfgOptCompressLevel,\n" - " cfgOptCompressLevelNetwork,\n" - "};\n"); - } - - FUNCTION_HARNESS_RETURN_VOID(); -} diff --git a/test/src/module/build/error/parse_test.py b/test/src/module/build/error/parse_test.py new file mode 100644 index 000000000..09e36c1b3 --- /dev/null +++ b/test/src/module/build/error/parse_test.py @@ -0,0 +1,57 @@ +"""Test Error Parse.""" + +#################################################################################################################################### +import os +import tempfile + +from harness.test import * + +from common.error import * +from common.storage import file_write +from error.parse import * + + +#################################################################################################################################### +def _parse(error): + """Parse an error declaration.""" + + with tempfile.TemporaryDirectory() as path: + file_write(os.path.join(path, "build/error.yaml"), error) + + return bld_err_parse(path) + + +#################################################################################################################################### +def _error(error): + """Parse an error declaration that is expected to fail and return the message.""" + + with assert_raises(ToolError) as exception: + _parse(error) + + return str(exception.exception) + + +#################################################################################################################################### +def test_error_parse(): + """An error is a code on its own or a code with the attributes that go with it.""" + + err_list = _parse("assert:\n code: 25\n fatal: true\nchecksum: 26\nfile-open: 41\n") + + # Errors keep the order they were declared in, since the generated array is indexed by nothing but position + assert_equal( + [(err.name, err.code, err.fatal) for err in err_list], + [("assert", 25, True), ("checksum", 26, False), ("file-open", 41, False)], + ) + + +#################################################################################################################################### +def test_error_parse_error(): + """An error declaration that cannot be honored is reported.""" + + assert_equal(_error("bogus:\n bogus: 1\n"), "unknown error definition 'bogus'") + assert_equal(_error("bogus:\n fatal: true\n"), "error 'bogus' requires a code") + assert_equal(_error("bogus: x\n"), "error 'bogus' code 'x' is not an integer") + + # A code must be inside the range the C reserves for the errors declared here + assert_equal(_error("bogus: 24\n"), "error 'bogus' code must be >= 25 and <= 125") + assert_equal(_error("bogus: 126\n"), "error 'bogus' code must be >= 25 and <= 125") diff --git a/test/src/module/build/error/render_test.py b/test/src/module/build/error/render_test.py new file mode 100644 index 000000000..33621e724 --- /dev/null +++ b/test/src/module/build/error/render_test.py @@ -0,0 +1,104 @@ +"""Test Error Render. + +The whole of both generated files is checked here, since they are short enough to read and what matters is the exact C that comes out.""" + +#################################################################################################################################### +import os +import tempfile + +from harness.test import * + +from common.render import COMMENT_BLOCK_BEGIN, COMMENT_BLOCK_END +from common.storage import file_read, file_write +from error.parse import bld_err_parse +from error.render import * + +# Header every generated file begins with +HEADER = """%s +Error Type Definition + +Automatically generated by 'build.py error' -- do not modify directly. +%s +""" % ( + COMMENT_BLOCK_BEGIN, + COMMENT_BLOCK_END, +) + + +#################################################################################################################################### +def _render(error): + """Render an error declaration and return what was written.""" + + with tempfile.TemporaryDirectory() as path: + file_write(os.path.join(path, "build/error.yaml"), error) + bld_err_render(path, bld_err_parse(path)) + + return ( + file_read(os.path.join(path, "src/common/error/error.auto.h")), + file_read(os.path.join(path, "src/common/error/error.auto.c.inc")), + ) + + +#################################################################################################################################### +def test_error_render(): + """An error is declared, defined, and listed under the C name built from its name.""" + + auto_h, auto_c = _render("assert:\n code: 25\n fatal: true\nfile-open: 41\n") + + assert_equal( + auto_h, + HEADER + + """#ifndef COMMON_ERROR_ERROR_AUTO_H +#define COMMON_ERROR_ERROR_AUTO_H + +%s +Error type declarations +%s +ERROR_DECLARE(AssertError); +ERROR_DECLARE(FileOpenError); + +#endif +""" + % (COMMENT_BLOCK_BEGIN, COMMENT_BLOCK_END), + ) + + # The code is aligned so the list reads as a column, and a fatal error is one the code itself is wrong about + assert_equal( + auto_c, + HEADER + + """ +%s +Error type definitions +%s +ERROR_DEFINE( 25, AssertError, true, RuntimeError); +ERROR_DEFINE( 41, FileOpenError, false, RuntimeError); + +%s +Error type array +%s +static const ErrorType *const errorTypeList[] = +{ + &AssertError, + &FileOpenError, + NULL, +}; +""" + % (COMMENT_BLOCK_BEGIN, COMMENT_BLOCK_END, COMMENT_BLOCK_BEGIN, COMMENT_BLOCK_END), + ) + + +#################################################################################################################################### +def test_error_render_write(): + """A generated file is only written when it changes, so an unchanged one does not trigger a rebuild.""" + + with tempfile.TemporaryDirectory() as path: + file_write(os.path.join(path, "build/error.yaml"), "assert: 25\n") + + bld_err_render(path, bld_err_parse(path)) + + path_auto_h = os.path.join(path, "src/common/error/error.auto.h") + time_before = os.stat(path_auto_h).st_mtime_ns + + bld_err_render(path, bld_err_parse(path)) + + assert_equal(os.stat(path_auto_h).st_mtime_ns, time_before) diff --git a/test/src/module/build/errorTest.c b/test/src/module/build/errorTest.c deleted file mode 100644 index 6a854e814..000000000 --- a/test/src/module/build/errorTest.c +++ /dev/null @@ -1,110 +0,0 @@ -/*********************************************************************************************************************************** -Test Build Error -***********************************************************************************************************************************/ -#include "harness/storage.h" - -/*********************************************************************************************************************************** -Test Run -***********************************************************************************************************************************/ -static void -testRun(void) -{ - FUNCTION_HARNESS_VOID(); - - // Create default storage object for testing - Storage *storageTest = storagePosixNewP(TEST_PATH_STR, .write = true); - - // ***************************************************************************************************************************** - if (testBegin("bldErrParse() and bldErrRender()")) - { - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("parse errors"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/error/error.yaml", - "assert: 24\n"); - - TEST_ERROR(bldErrParse(storageTest), FormatError, "error 'assert' code must be >= 25 and <= 125"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/error/error.yaml", - "assert: 126\n"); - - TEST_ERROR(bldErrParse(storageTest), FormatError, "error 'assert' code must be >= 25 and <= 125"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/error/error.yaml", - "assert:\n" - " bogus: 25"); - - TEST_ERROR(bldErrParse(storageTest), FormatError, "unknown error definition 'bogus'"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("parse and render error"); - - HRN_STORAGE_PUT_Z( - storageTest, "src/build/error/error.yaml", - "assert:\n" - " code: 25\n" - " fatal: true\n" - "option-invalid: 31\n" - "runtime: 122\n"); - - TEST_RESULT_VOID(bldErrRender(storageTest, bldErrParse(storageTest)), "parse and render"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("check error.auto.h"); - - TEST_STORAGE_GET( - storageTest, - "src/common/error/error.auto.h", - COMMENT_BLOCK_BEGIN "\n" - "Error Type Definition\n" - "\n" - "Automatically generated by 'build-code error' -- do not modify directly.\n" - COMMENT_BLOCK_END "\n" - "#ifndef COMMON_ERROR_ERROR_AUTO_H\n" - "#define COMMON_ERROR_ERROR_AUTO_H\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Error type declarations\n" - COMMENT_BLOCK_END "\n" - "ERROR_DECLARE(AssertError);\n" - "ERROR_DECLARE(OptionInvalidError);\n" - "ERROR_DECLARE(RuntimeError);\n" - "\n" - "#endif\n"); - - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("check error.auto.c.inc"); - - TEST_STORAGE_GET( - storageTest, - "src/common/error/error.auto.c.inc", - COMMENT_BLOCK_BEGIN "\n" - "Error Type Definition\n" - "\n" - "Automatically generated by 'build-code error' -- do not modify directly.\n" - COMMENT_BLOCK_END "\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Error type definitions\n" - COMMENT_BLOCK_END "\n" - "ERROR_DEFINE( 25, AssertError, true, RuntimeError);\n" - "ERROR_DEFINE( 31, OptionInvalidError, false, RuntimeError);\n" - "ERROR_DEFINE(122, RuntimeError, false, RuntimeError);\n" - "\n" - COMMENT_BLOCK_BEGIN "\n" - "Error type array\n" - COMMENT_BLOCK_END "\n" - "static const ErrorType *const errorTypeList[] =\n" - "{\n" - " &AssertError,\n" - " &OptionInvalidError,\n" - " &RuntimeError,\n" - " NULL,\n" - "};\n"); - } - - FUNCTION_HARNESS_RETURN_VOID(); -} diff --git a/test/src/module/build/help/parse_test.py b/test/src/module/build/help/parse_test.py new file mode 100644 index 000000000..e89af4249 --- /dev/null +++ b/test/src/module/build/help/parse_test.py @@ -0,0 +1,293 @@ +"""Test Help Parse. + +The help is checked against the configuration, so both are written out here. The configuration is the smallest one that has an option +documented in each of the three places help can be: a configuration section, the command line, and a single command.""" + +#################################################################################################################################### +import os +import tempfile + +from harness.test import * + +from common.error import * +from common.storage import file_write +from config.parse import bld_cfg_parse +from help.parse import * + +# Where the help declaration lives, which the tools tell the parser rather than the parser assuming +PATH_HELP = "doc/xml/reference.xml" + +CONFIG = """command: + backup: + command-role: + local: {} + help: {} + restore: {} + version: {} + +optionGroup: + pg: {} + +option: + buffer-size: + section: global + type: size + + force: + type: boolean + + stanza: + type: string + + set: + type: string + command: + restore: {} + + process: + type: integer + command-role: + local: {} + command: + backup: {} +""" + +# Help for everything the configuration declares. Buffer size is documented in a configuration section, force and stanza on the +# command line, and set only by the command that takes it. Process has no help at all, since no user can give it. +HELP = """ + + Configuration description. + + +

Configuration introduction.

+ + + + + +

Section introduction.

+ + + + + Buffer size. + + +

Buffer size description.

+ + + 1MiB + 32KiB + + + + + + + + Command description. + + +

Command introduction.

+ + + + + + + + + + + + + Restore summary. + + +

Restore description.

+ + + + + + + + + Backup summary. + + +

Backup description.

+ + + + + Help summary. + + +

Help description.

+ + + + + Version summary. + + +

Version description.

+ + + + + +""" + + +#################################################################################################################################### +def _parse(help=HELP, config=CONFIG, detail=True): + """Parse a help declaration against a configuration.""" + + with tempfile.TemporaryDirectory() as path: + file_write(os.path.join(path, "build/config.yaml"), config) + file_write(os.path.join(path, PATH_HELP), help) + + return bld_hlp_parse(os.path.join(path, PATH_HELP), bld_cfg_parse(path), detail) + + +#################################################################################################################################### +def _error(help=HELP, config=CONFIG): + """Parse a help declaration that is expected to fail and return the message.""" + + with assert_raises(ToolError) as error: + _parse(help, config) + + return str(error.exception) + + +#################################################################################################################################### +def _find(item_list, name): + """Find an entry in a list by name.""" + + return next(item for item in item_list if item.name == name) + + +#################################################################################################################################### +def test_help_parse(): + """Help is read for every command and option, wherever it is documented.""" + + bld_hlp = _parse() + + assert_equal(bld_hlp.cmd_title, "Command Reference") + assert_equal(bld_hlp.cmd_description, "Command description.") + assert_equal(bld_hlp.opt_title, "Configuration Reference") + assert_equal(bld_hlp.opt_description, "Configuration description.") + + # The introductions are left as nodes, since the same nodes are rendered as console text and as documentation + assert_equal(bld_hlp.cmd_introduction.tag, "text") + assert_equal(bld_hlp.opt_introduction.tag, "text") + + # Commands are sorted, whatever order they were documented in + assert_equal([cmd.name for cmd in bld_hlp.cmd_list], ["backup", "help", "restore", "version"]) + + cmd = _find(bld_hlp.cmd_list, "restore") + + assert_equal(cmd.title, "Restore") + assert_equal(cmd.summary.tag, "summary") + + # An option a command documents differently is under the command rather than in the option list + assert_equal([opt.name for opt in cmd.opt_list], ["set"]) + assert_is_none(_find(bld_hlp.cmd_list, "backup").opt_list) + + # Options are sorted and are found wherever they were documented + assert_equal([opt.name for opt in bld_hlp.opt_list], ["buffer-size", "force", "stanza"]) + + # An option documented in a configuration section takes the section it is documented in + opt = _find(bld_hlp.opt_list, "buffer-size") + + assert_equal(opt.section, "general") + assert_equal(opt.title, "Buffer Size") + assert_equal(opt.example_list, ["1MiB", "32KiB"]) + + # An option documented on the command line is in no section, unless it says which section it belongs to + assert_is_none(_find(bld_hlp.opt_list, "force").section) + assert_is_none(_find(bld_hlp.opt_list, "force").example_list) + assert_equal(_find(bld_hlp.opt_list, "stanza").section, "stanza") + + +#################################################################################################################################### +def test_help_parse_section(): + """A configuration section is where the options that can be set in the configuration file are documented.""" + + bld_hlp = _parse() + + assert_equal([(section.id, section.name) for section in bld_hlp.sct_list], [("general", "General")]) + assert_equal(bld_hlp.sct_list[0].introduction.tag, "text") + + +#################################################################################################################################### +def test_help_parse_detail(): + """The text only the documentation renders is not required when it is not going to be rendered.""" + + help = HELP.replace("Command description.", "") + help = help.replace( + """ +

Command introduction.

+ +""", + "", + ) + help = help.replace( + """ +

Section introduction.

+ +""", + "", + ) + + bld_hlp = _parse(help, detail=False) + + assert_is_none(bld_hlp.cmd_description) + assert_is_none(bld_hlp.cmd_introduction) + assert_is_none(bld_hlp.sct_list[0].introduction) + + # It is required when it is + assert_in("unable to find child 'text' in node 'config-section'", _error(help)) + + +#################################################################################################################################### +def test_help_parse_error(): + """Anything the user can reach must have help, since help that is missing is only found by looking for it.""" + + # A command with no help at all + assert_equal( + _error(HELP.replace('', '')), + "command 'backup' must have help", + ) + + # An option with no help of its own must be documented by every command that takes it + assert_equal( + _error(HELP.replace('