diff --git a/build/lib/postgres/parse.py b/build/lib/postgres/parse.py index 2c897a607..ebb5cf0db 100644 --- a/build/lib/postgres/parse.py +++ b/build/lib/postgres/parse.py @@ -26,6 +26,11 @@ _DEFINE_VERSION = "PG_VERSION" _DEFINE_CATALOG_MAX = "CATALOG_VERSION_NO_MAX" _DEFINE_EXTRA_LIST = (_DEFINE_CATALOG_MAX, _DEFINE_VERSION) +# What marks a macro as one that captures a value rather than as one of the functions the interface collects. Both take a +# version, so the name is what tells them apart, and the constant a value is captured as names the member it is carried in. +_VALUE_MARKER = "_VALUE_" +_VALUE_NAME = re.compile(r"(\w+)##version\s*=") + # A name in C, and the comments that are removed before looking for one so that a comment naming a type does not read as a use of it _NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") _COMMENT = re.compile(r"/\*.*?\*/|//[^\n]*", re.S) @@ -67,16 +72,27 @@ class BldPgEntity: self.version_map = version_map # Branch each version resolves to, or _BRANCH_ABSENT for a version it is missing from +#################################################################################################################################### +class BldPgValue: + """A value an interface carries as data.""" + + def __init__(self, macro, name): + self.macro = macro # Macro capturing it for a version, which is rendered and shared the same way a function is + self.name = name # Constant it is captured as, which also names the member it is carried in + + #################################################################################################################################### class BldPgInterface: """An interface that is generated from the vendored headers.""" - def __init__(self, module, description, path_intern, include, prefix, path_render): + def __init__(self, module, description, path_intern, include, prefix, vendor, unreleased, path_render): self.module = module # Generator that renders it, which the generated file names as what wrote it self.description = description # What the generated file says it is self.path_intern = path_intern # Header the interface macros are declared in self.include = include # How the generated code includes that header self.prefix = prefix # Prefix of the functions the macros define, e.g. "pgInterface" + self.vendor = vendor # Does it declare the vendored types, or use the ones another interface declared? + self.unreleased = unreleased # Does it carry the mark for a version that has not been released? self.path_render = path_render # Where the interface is rendered @property @@ -93,6 +109,8 @@ BLD_PG_INTERFACE = BldPgInterface( "src/postgres/interface/version.intern.h", "postgres/interface/version.intern.h", "pgInterface", + True, + True, "src/postgres/interface.auto.c.inc", ) @@ -103,6 +121,8 @@ BLD_PG_INTERFACE_HARNESS = BldPgInterface( "test/src/harness/postgres/version.intern.h", "harness/postgres/version.intern.h", "hrnPgInterface", + False, + False, "test/src/harness/postgres/interface.auto.c.inc", ) @@ -111,12 +131,14 @@ BLD_PG_INTERFACE_HARNESS = BldPgInterface( class BldPg: """The PostgreSQL interface declaration.""" - def __init__(self, pg_list, type_list, define_list, function_list, function_version, interface): + def __init__(self, pg_list, type_list, define_list, function_list, function_version, value_list, macro_list, interface): self.pg_list = pg_list # Supported versions, oldest first self.type_list = type_list # Interface types, sorted self.define_list = define_list # Interface defines, sorted self.function_list = function_list # Functions defined by macros, in the order they are declared self.function_version = function_version # Version whose rendering of a function each version uses, by function + self.value_list = value_list # Constants the values are captured as, which also name the members they are carried in + self.macro_list = macro_list # Every macro the interface header declares, which says what it has to render self.interface = interface # Interface the functions were read for, which is the one that gets rendered @@ -194,9 +216,9 @@ def _define_list(header): def _function_list(header): """Scan the interface functions out of a header, which are the macros taking the version to render one for. - A macro that does not take a version is a helper the function macros are written with rather than a function the interface has, - so it is not rendered. It is still followed when working out what a function depends on, since a type it names is a type the - function that uses it reaches.""" + A macro that does not take a version is a value or a helper the function macros are written with rather than a function the + interface has, so it is not rendered. It is still followed when working out what a function depends on, since a type it names is + a type the function that uses it reaches.""" result = [] @@ -206,12 +228,35 @@ def _function_list(header): if define is None or not line.strip()[len("#define ") + len(define) :].startswith("(version)"): continue + if _VALUE_MARKER in define: + continue + if define not in result: result.append(define) return result +#################################################################################################################################### +def _value_list(macro_map): + """Scan the values an interface carries out of the macros capturing them, in the order they are declared. + + Each is captured by a macro of its own, as a constant named for the version, e.g. pgInterfaceControlVersion##version. That name + is also what says which member the value is carried in, e.g. controlVersion. A macro of its own is what lets a value that does + not vary between two versions be captured once for both, the way a function that does not vary is rendered once for both.""" + + result = [] + + for macro, body in macro_map.items(): + if _VALUE_MARKER not in macro: + continue + + for name in _VALUE_NAME.findall(body): + result.append(BldPgValue(macro, name)) + + return result + + #################################################################################################################################### def _type_list(line_list): """Scan the types out of the lines of a header. @@ -475,10 +520,20 @@ def bld_pg_parse(path_repo, interface=BLD_PG_INTERFACE): type_list = sorted(_type_list(header_vendor.split("\n"))) define_list = sorted(_define_list(header_vendor) + list(_DEFINE_EXTRA_LIST)) - # Functions are defined as macros, which each interface expands for its own version + # Functions are defined as macros, which each interface expands for its own version, and values as macros it reads function_list = _function_list(header_intern) + macro_map = _macro_map(header_intern) + macro_list = _define_list(header_intern) # A function only has to be expanded once for each set of branches it reaches, so work out which versions can share one - function_version = _function_version(pg_list, _entity_map(header_vendor, pg_list), _macro_map(header_intern), function_list) + value_list = _value_list(macro_map) - return BldPg(pg_list, type_list, define_list, function_list, function_version, interface) + # A value is rendered and shared the same way a function is, so which versions can share one is worked out for both + function_version = _function_version( + pg_list, + _entity_map(header_vendor, pg_list), + macro_map, + function_list + list(dict.fromkeys(value.macro for value in value_list)), + ) + + return BldPg(pg_list, type_list, define_list, function_list, function_version, value_list, macro_list, interface) diff --git a/build/lib/postgres/render.py b/build/lib/postgres/render.py index 6461e42fc..ac24acff6 100644 --- a/build/lib/postgres/render.py +++ b/build/lib/postgres/render.py @@ -4,9 +4,12 @@ Each version gets a block that includes the vendored header again with every nam translation unit carries an interface for every version it supports. The interfaces are rendered newest first so the most likely match is found first at run time. -A block only renders the functions that no older version already renders the same code for, since a function whose types and defines -all resolve to the same branches for two versions would compile to the same thing twice. The struct then points both versions at the -one that was rendered, so what is shared is visible where it is used. +A block only renders the functions that no other version already renders the same code for, since a function whose types and +defines all resolve to the same branches for two versions would compile to the same thing twice. The struct then points both +versions at the one that was rendered, so what is shared is visible where it is used. + +The values a version carries are macros of the vendored header, which have a value only inside its block, so the block captures +them as constants named for the version and the struct names those. The interface being rendered is whichever one the declaration was parsed for, so the binary and the test harness are rendered by the same code from the same versions.""" @@ -37,6 +40,18 @@ _PATH_SYSTEM_ID = "test/src/harness/postgres.h" # Column the comment naming the versions that share a rendering is aligned at, which is the column a #define value is aligned at _SHARE_COLUMN = 68 +# Member marking a version that has not been released, which the declaration says rather than the interface header +_UNRELEASED_MEMBER = "unreleased" + +# Macro capturing the values of a version as constants named for it, which is rendered in the block since that is the only place +# they have a value +_VALUE_MACRO = "%s_VALUE" + +# Macro matching a version against an interface, which is rendered once after the versions when the interface declares it. The +# range form is rendered when a version has not been released, since only then does an interface accept more than one value. +_MATCH_MACRO = "%s_MATCH" +_MATCH_MACRO_RANGE = "%s_MATCH_RANGE" + #################################################################################################################################### def _version_no_dot(version): @@ -70,7 +85,7 @@ def _version_suffix(bld_pg, function, version): #################################################################################################################################### def _function_name(define): - """The interface function a macro defines, e.g. "PG_INTERFACE_CONTROL_IS" becomes "pgInterfaceControlIs". + """The interface function a macro defines, e.g. "PG_INTERFACE_CONTROL" becomes "pgInterfaceControl". A harness macro reads the same way, e.g. "HRN_PG_INTERFACE_CONTROL" becomes "hrnPgInterfaceControl".""" @@ -83,6 +98,37 @@ def _function_name(define): return result +#################################################################################################################################### +def _render_macro_list(bld_pg, macro_list, pg): + """Macros of a list that a version renders, which are the ones no other version already renders the same code for.""" + + return [macro for macro in macro_list if bld_pg.function_version[macro][pg.version] == pg.version] + + +#################################################################################################################################### +def _render_expand(bld_pg, macro_list, pg): + """Render the macros a version expands, each saying which other versions share the rendering.""" + + result = "" + + for macro in macro_list: + expand = "%s(%s);" % (macro, _version_suffix(bld_pg, macro, pg.version)) + + # Versions using this rendering rather than one of their own, which says why they have no rendering of it + share_list = [ + pg_share.version + for pg_share in bld_pg.pg_list + if pg_share.version != pg.version and bld_pg.function_version[macro][pg_share.version] == pg.version + ] + + if share_list != []: + expand += "%*s// Shared with %s" % (_SHARE_COLUMN - len(expand), "", ", ".join(share_list)) + + result += expand + "\n" + + return result + + #################################################################################################################################### def _render_interface_auto_c(bld_pg): """Render the interface, which is one interface per version plus the struct that finds them.""" @@ -90,51 +136,59 @@ def _render_interface_auto_c(bld_pg): interface = bld_pg.interface result = bld_header(interface.module, interface.description) - # Interfaces, newest first + # Interfaces, newest first so the most likely match is found first at run time for pg in reversed(bld_pg.pg_list): version_no_dot = _version_no_dot(pg.version) - # Functions this version renders, which is the ones no older version already renders the same code for - function_list = [ - function for function in bld_pg.function_list if bld_pg.function_version[function][pg.version] == pg.version - ] - - # A version that shares every one of its functions has nothing to render, so it gets no block at all - if function_list == []: - continue + # Values and functions this version renders, which are the ones no other version already renders the same code for + value_list = _render_macro_list(bld_pg, list(dict.fromkeys(value.macro for value in bld_pg.value_list)), pg) + function_list = _render_macro_list(bld_pg, bld_pg.function_list, pg) result += "\n" + bld_comment_block("PostgreSQL %s interface" % pg.version) - result += bld_define("PG_VERSION", "PG_VERSION_%s" % version_no_dot) + "\n\n" + + # Only the interface declaring the vendored types sets the version they are declared for. The other renames them to + # what that one declared, which is why they are named the same way here. + if interface.vendor: + result += bld_define("PG_VERSION", "PG_VERSION_%s" % version_no_dot) + "\n\n" for type in bld_pg.type_list: result += bld_define(type, "%s_%s" % (type, version_no_dot)) + "\n" # An unreleased version has no catalog version of its own yet, so it accepts any catalog version up to the maximum - if not pg.release: + if interface.vendor and not pg.release: result += "\n#define CATALOG_VERSION_NO_MAX\n" result += '\n#include "%s"\n\n' % interface.include - for function in function_list: - expand = "%s(%s);" % (function, _version_suffix(bld_pg, function, pg.version)) + # Values this version captures as constants named for it, so the struct at the end can name them + result += _render_expand(bld_pg, value_list, pg) - # Versions using this rendering rather than one of their own, which says why they have no rendering of it - share_list = [ - pg_share.version - for pg_share in bld_pg.pg_list - if pg_share.version != pg.version and bld_pg.function_version[function][pg_share.version] == pg.version - ] + # A value is read where a function is called, so what a version captures is separated from what it renders + if value_list != [] and function_list != []: + result += "\n" - if share_list != []: - expand += "%*s// Shared with %s" % (_SHARE_COLUMN - len(expand), "", ", ".join(share_list)) - - result += expand + "\n" + result += _render_expand(bld_pg, function_list, pg) # Undefine everything the interface defined so the next one starts clean result += "\n" + "".join("#undef %s\n" % type for type in bld_pg.type_list) - result += "\n" + "".join("#undef %s\n" % define for define in bld_pg.define_list) + + # Only the interface declaring the vendored types defines what they define, so only it has them to undefine + if interface.vendor: + result += "\n" + "".join("#undef %s\n" % define for define in bld_pg.define_list) + result += "\n" + "".join("#undef %s\n" % function for function in bld_pg.function_list) + # Matches, rendered once each in the form that fits the versions that were rendered. Only a version that has not been released + # accepts a range of values, so the range form is rendered only while there is one. + release = all(pg.release for pg in bld_pg.pg_list) + + for macro in [macro for macro in bld_pg.macro_list if macro.endswith(_MATCH_MACRO % "")]: + prefix = macro[: -len(_MATCH_MACRO % "")] + member = _function_name(prefix)[len(interface.prefix) :] + + result += "\n" + bld_comment_block("PostgreSQL %s match" % (member[:1].lower() + member[1:])) + result += "%s();\n" % ((_MATCH_MACRO if release else _MATCH_MACRO_RANGE) % prefix) + # Interface struct, newest first so the most likely match is found first result += "\n" + bld_comment_block( "PostgreSQL interface struct\n\nA function shared by more than one version is named for the range it covers, so a version" @@ -143,7 +197,28 @@ def _render_interface_auto_c(bld_pg): result += "static const %s %s[] =\n{\n" % (interface.type, interface.prefix) for pg in reversed(bld_pg.pg_list): - result += " {\n .version = PG_VERSION_%s,\n\n" % _version_no_dot(pg.version) + version_no_dot = _version_no_dot(pg.version) + + result += " {\n .version = PG_VERSION_%s,\n" % version_no_dot + + # A version that has not been released accepts a range of catalog versions rather than only its own + if interface.unreleased and not pg.release: + result += " .%s = true,\n" % _UNRELEASED_MEMBER + + result += "\n" + + # An interface reading the values from another one carries none of its own, so there is no group to separate + if bld_pg.value_list != []: + for value in bld_pg.value_list: + member = value.name[len(interface.prefix) :] + + result += " .%s = %s%s,\n" % ( + member[:1].lower() + member[1:], + value.name, + _version_suffix(bld_pg, value.macro, bld_pg.function_version[value.macro][pg.version]), + ) + + result += "\n" for function in bld_pg.function_list: name = _function_name(function) diff --git a/src/postgres/interface.c b/src/postgres/interface.c index a2852fc82..b12de32df 100644 --- a/src/postgres/interface.c +++ b/src/postgres/interface.c @@ -62,34 +62,31 @@ typedef struct PgInterface // Version of PostgreSQL supported by this interface unsigned int version; - // Does pg_control match this version of PostgreSQL? - bool (*controlIs)(const uint8_t *); + // Control version and catalog version that pg_control matches for this version of PostgreSQL. A version that has not been + // released has no catalog version of its own yet, so it accepts a range of them rather than only its own. + uint32_t controlVersion; + uint32_t catalogVersion; + bool unreleased; + + // Magic that the WAL header matches for this version of PostgreSQL + uint16_t walMagic; + + // Offset of the crc in pg_control + size_t controlCrcOffset; // Convert pg_control to a common data structure PgControl (*control)(const uint8_t *); - // Get control crc offset - size_t (*controlCrcOffset)(void); - // Invalidate control checkpoint void (*controlCheckpointInvalidate)(uint8_t *); - // Get the control version for this version of PostgreSQL - uint32_t (*controlVersion)(void); - - // Does the WAL header match this version of PostgreSQL? - bool (*walIs)(const uint8_t *); - // Convert WAL header to a common data structure PgWal (*wal)(const uint8_t *); } PgInterface; -// Include auto-generated interfaces -#include "postgres/interface.auto.c.inc" - /*********************************************************************************************************************************** -These pg_control fields are common to all versions of PostgreSQL, so we can use them to generate error messages when the pg_control -version cannot be found. +These pg_control fields are common to all versions of PostgreSQL, so we can use them to match an interface and to generate error +messages when the pg_control version cannot be found. ***********************************************************************************************************************************/ typedef struct PgControlCommon { @@ -98,6 +95,9 @@ typedef struct PgControlCommon uint32_t catalogVersion; } PgControlCommon; +// Include auto-generated interfaces +#include "postgres/interface.auto.c.inc" + /*********************************************************************************************************************************** Get the interface for a PostgreSQL version ***********************************************************************************************************************************/ @@ -203,7 +203,7 @@ pgControlCrcValidate(const Buffer *const controlFile, const PgInterface *const i ASSERT(controlFile != NULL); ASSERT(interface != NULL); - size_t result = interface->controlCrcOffset(); + size_t result = interface->controlCrcOffset; do { @@ -283,9 +283,11 @@ pgControlFromBuffer(const Buffer *const controlFile, const String *const pgVersi interface = pgInterfaceVersion(pgVersionFromStr(pgVersionForce)); else { + const PgControlCommon *const controlCommon = (const PgControlCommon *)bufPtrConst(controlFile); + for (unsigned int interfaceIdx = 0; interfaceIdx < LENGTH_OF(pgInterface); interfaceIdx++) { - if (pgInterface[interfaceIdx].controlIs(bufPtrConst(controlFile))) + if (pgInterfaceControlMatch(&pgInterface[interfaceIdx], controlCommon)) { interface = &pgInterface[interfaceIdx]; break; @@ -295,8 +297,6 @@ pgControlFromBuffer(const Buffer *const controlFile, const String *const pgVersi // If the version was not found then error with the control and catalog version that were found if (interface == NULL) { - const PgControlCommon *controlCommon = (const PgControlCommon *)bufPtrConst(controlFile); - THROW_FMT( VersionNotSupportedError, "unexpected control version = %u and catalog version = %u\n" @@ -307,8 +307,6 @@ pgControlFromBuffer(const Buffer *const controlFile, const String *const pgVersi } // Get info from the control file - ASSERT(pgVersionForce != NULL || interface->controlIs(bufPtrConst(controlFile))); - PgControl result = interface->control(bufPtrConst(controlFile)); result.version = interface->version; @@ -460,7 +458,7 @@ pgControlVersion(const unsigned int pgVersion) FUNCTION_TEST_PARAM(UINT, pgVersion); FUNCTION_TEST_END(); - FUNCTION_TEST_RETURN(UINT32, pgInterfaceVersion(pgVersion)->controlVersion()); + FUNCTION_TEST_RETURN(UINT32, pgInterfaceVersion(pgVersion)->controlVersion); } /*********************************************************************************************************************************** @@ -499,7 +497,7 @@ pgWalFromBuffer(const Buffer *const walBuffer, const String *const pgVersionForc { for (unsigned int interfaceIdx = 0; interfaceIdx < LENGTH_OF(pgInterface); interfaceIdx++) { - if (pgInterface[interfaceIdx].walIs(bufPtrConst(walBuffer))) + if (((const PgWalCommon *)bufPtrConst(walBuffer))->magic == pgInterface[interfaceIdx].walMagic) { interface = &pgInterface[interfaceIdx]; break; @@ -519,8 +517,6 @@ pgWalFromBuffer(const Buffer *const walBuffer, const String *const pgVersionForc } // Get info from the control file - ASSERT(pgVersionForce != NULL || interface->walIs(bufPtrConst(walBuffer))); - PgWal result = interface->wal(bufPtrConst(walBuffer)); result.version = interface->version; diff --git a/src/postgres/interface/version.intern.h b/src/postgres/interface/version.intern.h index f179e42d3..84f1f9fe5 100644 --- a/src/postgres/interface/version.intern.h +++ b/src/postgres/interface/version.intern.h @@ -10,39 +10,64 @@ These macros should be kept as simple as possible, with most of the logic contai #include "postgres/version.h" /*********************************************************************************************************************************** -Determine if the supplied pg_control is for this version of PostgreSQL. When CATALOG_VERSION_NO_MAX is defined then the catalog will -be accepted as a range that lasts until the end of the encoded year. This allows pgBackRest to work with PostgreSQL during the -alpha/beta/rc period without needing to be updated, unless of course the actual interface changes. +Values that vary by version + +Each is captured as a constant named for the version so that the interface struct, which is built after every version has +undefined what it defined, can still name it. A define cannot do this because it is expanded where it is used rather than where +it is made, by which time the value it was given is gone. + +One macro per value so that a value which does not vary between two versions is captured once for both, the same way a function +that does not vary is rendered once for both. ***********************************************************************************************************************************/ -#ifdef CATALOG_VERSION_NO_MAX +// Control version that pg_control matches for this version of PostgreSQL +#define PG_INTERFACE_VALUE_CONTROL_VERSION(version) \ + enum {pgInterfaceControlVersion##version = PG_CONTROL_VERSION} -#define PG_INTERFACE_CONTROL_IS(version) \ +// Catalog version that pg_control matches for this version of PostgreSQL +#define PG_INTERFACE_VALUE_CATALOG_VERSION(version) \ + enum {pgInterfaceCatalogVersion##version = CATALOG_VERSION_NO} + +// Magic that the WAL header matches for this version of PostgreSQL +#define PG_INTERFACE_VALUE_WAL_MAGIC(version) \ + enum {pgInterfaceWalMagic##version = XLOG_PAGE_MAGIC} + +// Offset of the crc in pg_control, which is also the length of the part the crc is calculated over since the crc is last +#define PG_INTERFACE_VALUE_CONTROL_CRC_OFFSET(version) \ + enum {pgInterfaceControlCrcOffset##version = offsetof(ControlFileData, crc)} + +/*********************************************************************************************************************************** +Determine whether pg_control matches an interface + +Rendered once after the versions, in whichever form fits the versions that were rendered. The range form is rendered when one of +them has not been released, since only then is there an interface accepting more than a single catalog version. Rendering the form +that is used means there is never a comparison that no supported version can fail. + +An unreleased version accepts any catalog version until the end of the year the one it was built with encodes. +***********************************************************************************************************************************/ +#define PG_INTERFACE_CONTROL_MATCH() \ static bool \ - pgInterfaceControlIs##version(const uint8_t *controlFile) \ + pgInterfaceControlMatch(const PgInterface *const interface, const PgControlCommon *const control) \ { \ - ASSERT(controlFile != NULL); \ - \ - return \ - ((const ControlFileData *)controlFile)->pg_control_version == PG_CONTROL_VERSION && \ - ((const ControlFileData *)controlFile)->catalog_version_no >= CATALOG_VERSION_NO && \ - ((const ControlFileData *)controlFile)->catalog_version_no < (CATALOG_VERSION_NO / 100000 + 1) * 100000; \ + return control->controlVersion == interface->controlVersion && control->catalogVersion == interface->catalogVersion; \ } -#else - -#define PG_INTERFACE_CONTROL_IS(version) \ +#define PG_INTERFACE_CONTROL_MATCH_RANGE() \ static bool \ - pgInterfaceControlIs##version(const uint8_t *controlFile) \ + pgInterfaceControlMatch(const PgInterface *const interface, const PgControlCommon *const control) \ { \ - ASSERT(controlFile != NULL); \ + if (control->controlVersion != interface->controlVersion) \ + return false; \ \ - return \ - ((const ControlFileData *)controlFile)->pg_control_version == PG_CONTROL_VERSION && \ - ((const ControlFileData *)controlFile)->catalog_version_no == CATALOG_VERSION_NO; \ + if (interface->unreleased) \ + { \ + return \ + control->catalogVersion >= interface->catalogVersion && \ + control->catalogVersion < (interface->catalogVersion / 100000 + 1) * 100000; \ + } \ + \ + return control->catalogVersion == interface->catalogVersion; \ } -#endif - /*********************************************************************************************************************************** Read the version specific pg_control into a general data structure ***********************************************************************************************************************************/ @@ -64,16 +89,6 @@ Read the version specific pg_control into a general data structure }; \ } -/*********************************************************************************************************************************** -Get control crc offset -***********************************************************************************************************************************/ -#define PG_INTERFACE_CONTROL_CRC_OFFSET(version) \ - static size_t \ - pgInterfaceControlCrcOffset##version(void) \ - { \ - return offsetof(ControlFileData, crc); \ - } - /*********************************************************************************************************************************** Invalidate control checkpoint. PostgreSQL skips the first segment so any LSN in that segment is invalid. ***********************************************************************************************************************************/ @@ -84,28 +99,6 @@ Invalidate control checkpoint. PostgreSQL skips the first segment so any LSN in ((ControlFileData *)controlFile)->checkPoint = PG_CONTROL_CHECKPOINT_INVALID; \ } -/*********************************************************************************************************************************** -Get the control version -***********************************************************************************************************************************/ -#define PG_INTERFACE_CONTROL_VERSION(version) \ - static uint32_t \ - pgInterfaceControlVersion##version(void) \ - { \ - return PG_CONTROL_VERSION; \ - } - -/*********************************************************************************************************************************** -Determine if the supplied WAL is for this version of PostgreSQL -***********************************************************************************************************************************/ -#define PG_INTERFACE_WAL_IS(version) \ - static bool \ - pgInterfaceWalIs##version(const uint8_t *walFile) \ - { \ - ASSERT(walFile != NULL); \ - \ - return ((const XLogPageHeaderData *)walFile)->xlp_magic == XLOG_PAGE_MAGIC; \ - } - /*********************************************************************************************************************************** Read the version specific WAL header into a general data structure ***********************************************************************************************************************************/ diff --git a/test/define.yaml b/test/define.yaml index 8085bdd4b..423aa2030 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -655,7 +655,10 @@ unit: # -------------------------------------------------------------------------------------------------------------------------------- - name: postgres/interface total: 10 - harness: postgres + harness: + name: postgres + shim: + - postgres/interface coverage: - postgres/interface diff --git a/test/src/harness/postgres.c b/test/src/harness/postgres.c index b103a01b4..b7acfc7e4 100644 --- a/test/src/harness/postgres.c +++ b/test/src/harness/postgres.c @@ -9,6 +9,14 @@ Harness for PostgreSQL Interface #include "harness/debug.h" #include "harness/postgres.h" +/*********************************************************************************************************************************** +Include shimmed C modules + +The interface the code reads is shimmed so the harness can write with the types it declares rather than declaring them again, and so +the values a test gets by default are the ones the code matches on rather than a second copy of them. +***********************************************************************************************************************************/ +{[SHIM_MODULE]} + /*********************************************************************************************************************************** Interface definition ***********************************************************************************************************************************/ @@ -17,9 +25,6 @@ typedef struct HrnPgInterface // Version of PostgreSQL supported by this interface unsigned int version; - // Catalog version - unsigned int (*catalogVersion)(void); - // Create pg_control void (*control)(unsigned int, unsigned int, PgControl, uint8_t *); @@ -66,12 +71,12 @@ hrnPgCatalogVersion(unsigned int pgVersion) FUNCTION_HARNESS_PARAM(UINT, pgVersion); FUNCTION_HARNESS_END(); - FUNCTION_HARNESS_RETURN(UINT, hrnPgInterfaceVersion(pgVersion)->catalogVersion()); + FUNCTION_HARNESS_RETURN(UINT, pgInterfaceVersion(pgVersion)->catalogVersion); } /**********************************************************************************************************************************/ Buffer * -hrnPgControlToBuffer(const unsigned int controlVersion, const unsigned int crc, PgControl pgControl) +hrnPgControlToBuffer(unsigned int controlVersion, const unsigned int crc, PgControl pgControl) { FUNCTION_HARNESS_BEGIN(); FUNCTION_HARNESS_PARAM(UINT, controlVersion); @@ -87,7 +92,7 @@ hrnPgControlToBuffer(const unsigned int controlVersion, const unsigned int crc, pgControl.walSegmentSize == UINT_MAX ? 0 : (pgControl.walSegmentSize == 0 ? HRN_PG_WAL_SEGMENT_SIZE_DEFAULT : pgControl.walSegmentSize); pgControl.catalogVersion = - pgControl.catalogVersion == 0 ? hrnPgInterfaceVersion(pgControl.version)->catalogVersion() : pgControl.catalogVersion; + pgControl.catalogVersion == 0 ? pgInterfaceVersion(pgControl.version)->catalogVersion : pgControl.catalogVersion; pgControl.systemId = pgControl.systemId < 100 ? hrnPgSystemId(pgControl.version) + pgControl.systemId : pgControl.systemId; pgControl.checkpoint = pgControl.checkpoint == 0 ? 1 : pgControl.checkpoint; pgControl.timeline = pgControl.timeline == 0 ? 1 : pgControl.timeline; @@ -98,6 +103,9 @@ hrnPgControlToBuffer(const unsigned int controlVersion, const unsigned int crc, bufUsedSet(result, bufSize(result)); // Generate pg_control + if (controlVersion == 0) + controlVersion = pgInterfaceVersion(pgControl.version)->controlVersion; + hrnPgInterfaceVersion(pgControl.version)->control(controlVersion, crc, pgControl, bufPtr(result)); FUNCTION_HARNESS_RETURN(BUFFER, result); @@ -105,7 +113,7 @@ hrnPgControlToBuffer(const unsigned int controlVersion, const unsigned int crc, /**********************************************************************************************************************************/ void -hrnPgWalToBuffer(Buffer *const walBuffer, const unsigned int magic, PgWal pgWal) +hrnPgWalToBuffer(Buffer *const walBuffer, unsigned int magic, PgWal pgWal) { FUNCTION_HARNESS_BEGIN(); FUNCTION_HARNESS_PARAM(BUFFER, walBuffer); @@ -124,6 +132,9 @@ hrnPgWalToBuffer(Buffer *const walBuffer, const unsigned int magic, PgWal pgWal) pgWal.systemId = hrnPgSystemId(pgWal.version) + pgWal.systemId; // Generate WAL + if (magic == 0) + magic = pgInterfaceVersion(pgWal.version)->walMagic; + hrnPgInterfaceVersion(pgWal.version)->wal(magic, pgWal, bufPtr(walBuffer)); FUNCTION_HARNESS_RETURN_VOID(); diff --git a/test/src/harness/postgres/version.intern.h b/test/src/harness/postgres/version.intern.h index 93e783708..c71a28f5a 100644 --- a/test/src/harness/postgres/version.intern.h +++ b/test/src/harness/postgres/version.intern.h @@ -1,23 +1,12 @@ /*********************************************************************************************************************************** Harness for PostgreSQL Interface -Macros for building version-specific functions that write the pg_control and WAL a test needs, using the types in version.vendor.h. -This is the write side of the interface the code reads with postgres/interface/version.intern.h. +Macros for building version-specific functions that write the pg_control and WAL a test needs. This is the write side of the +interface the code reads with postgres/interface/version.intern.h. -The header is included again for each version, so it declares only what varies by version. Everything else the macros use is -included by the module that expands them. +The types are the ones that interface declares, which the harness reaches by shimming it rather than declaring them again, so a type +may only be named here if it is named there. Everything else the macros use is included by the module that expands them. ***********************************************************************************************************************************/ -#include "postgres/interface/version.vendor.h" - -/*********************************************************************************************************************************** -Get the catalog version -***********************************************************************************************************************************/ -#define HRN_PG_INTERFACE_CATALOG_VERSION(version) \ - static uint32_t \ - hrnPgInterfaceCatalogVersion##version(void) \ - { \ - return CATALOG_VERSION_NO; \ - } /*********************************************************************************************************************************** Create a pg_control file @@ -32,7 +21,7 @@ Create a pg_control file *(ControlFileData *)buffer = (ControlFileData) \ { \ .system_identifier = pgControl.systemId, \ - .pg_control_version = controlVersion == 0 ? PG_CONTROL_VERSION : controlVersion, \ + .pg_control_version = controlVersion, \ .catalog_version_no = pgControl.catalogVersion, \ .checkPoint = pgControl.checkpoint, \ .checkPointCopy = \ @@ -54,8 +43,8 @@ Create a WAL file static void \ hrnPgInterfaceWal##version(const unsigned int magic, const PgWal pgWal, uint8_t *const buffer) \ { \ - ((XLogLongPageHeaderData *)buffer)->std.xlp_magic = magic == 0 ? XLOG_PAGE_MAGIC : (uint16)magic; \ - ((XLogLongPageHeaderData *)buffer)->std.xlp_info = XLP_LONG_HEADER; \ + ((XLogLongPageHeaderData *)buffer)->std.xlp_magic = (uint16)magic; \ + ((XLogLongPageHeaderData *)buffer)->std.xlp_info = PG_WAL_LONG_HEADER; \ ((XLogLongPageHeaderData *)buffer)->xlp_sysid = pgWal.systemId; \ ((XLogLongPageHeaderData *)buffer)->xlp_seg_size = pgWal.size; \ } diff --git a/test/src/module/build/postgres/render_test.py b/test/src/module/build/postgres/render_test.py index 939f11401..03680504d 100644 --- a/test/src/module/build/postgres/render_test.py +++ b/test/src/module/build/postgres/render_test.py @@ -14,11 +14,21 @@ from postgres.render import * # The comment the vendored header divides itself with, one before each entity it declares SEPARATOR = "// " + "-" * 129 -# A vendored header with a type that never varies and a type that does, since what a function reaches decides what can be shared +# A vendored header with a type that never varies, a type that does, and a define the interface carries as a value, since what a +# function reaches decides what can be shared VENDOR = ( "// TransactionId type\n" + SEPARATOR + "\n" "typedef uint32 TransactionId;\n" "\n" + "// PG_CONTROL_VERSION define\n" + SEPARATOR + "\n" + "#if PG_VERSION > PG_VERSION_MAX\n" + "\n" + "#elif PG_VERSION >= PG_VERSION_96\n" + "\n" + "#define PG_CONTROL_VERSION 960\n" + "\n" + "#endif\n" + "\n" "// ControlFileData type\n" + SEPARATOR + "\n" "#if PG_VERSION > PG_VERSION_MAX\n" "\n" @@ -40,7 +50,8 @@ VENDOR = ( ) # One function reading the type that varies, so every version renders it, and one reading only the type that does not, so they -# share. The first is declared twice under a conditional, as the real header declares it, so which is rendered is a dependency too. +# share. The first is declared twice under a conditional, as the real header declares it, so which is rendered is a dependency +# too. The value macro is not a function, so it is rendered for every version and collected by none of them. INTERN = ( "#ifdef CATALOG_VERSION_NO_MAX\n" "\n" @@ -56,9 +67,14 @@ INTERN = ( "\n" "#define PG_INTERFACE_CONTROL_CRC_OFFSET(version)\\\n" " static size_t pgInterfaceControlCrcOffset##version(void) { return sizeof(TransactionId); }\n" + "\n" + "#define PG_INTERFACE_VALUE_CONTROL_VERSION(version) enum {pgInterfaceControlVersion##version = PG_CONTROL_VERSION}\n" + "\n" + "#define PG_INTERFACE_CONTROL_MATCH() static bool pgInterfaceControlMatch(void) { return true; }\n" + "#define PG_INTERFACE_CONTROL_MATCH_RANGE() static bool pgInterfaceControlMatch(void) { return false; }\n" ) -# An interface where every function is shared, which leaves the newer version with nothing of its own to render +# An interface with no function that varies, which leaves the newer version rendering nothing but its own interface INTERN_SHARE = ( "#define PG_INTERFACE_CONTROL_CRC_OFFSET(version)\\\n" " static size_t pgInterfaceControlCrcOffset##version(void) { return sizeof(TransactionId); }\n" @@ -148,20 +164,34 @@ def test_postgres_render_interface(): assert_in("PG_INTERFACE_CONTROL_CRC_OFFSET(96_10); // Shared with 10\n", interface) assert_not_in("PG_INTERFACE_CONTROL_CRC_OFFSET(10)", interface) - # The struct names each function after the macro that defines it, and a version that shares one names the version that rendered - # it rather than itself + # A version captures the values it carries in its own block, since that is the only place they have a value, and a value + # that does not vary is captured once for every version that shares it, the same way a function is rendered once + assert_in("PG_INTERFACE_VALUE_CONTROL_VERSION(96_10); // Shared with 10\n", interface) + assert_not_in("PG_INTERFACE_VALUE_CONTROL_VERSION(10)", interface) + + # Every version has been released, so the match that compares a single value is the one rendered + assert_in("PostgreSQL control match\n", interface) + assert_in("PG_INTERFACE_CONTROL_MATCH();\n", interface) + assert_not_in("PG_INTERFACE_CONTROL_MATCH_RANGE();", interface) + + # The struct names each value after the constant the block captured it as, and the member it is carried in after that same + # name. A function it shares names the version that rendered it rather than itself. assert_in( """static const PgInterface pgInterface[] = { { .version = PG_VERSION_10, + .controlVersion = pgInterfaceControlVersion96_10, + .controlIs = pgInterfaceControlIs10, .controlCrcOffset = pgInterfaceControlCrcOffset96_10, }, { .version = PG_VERSION_96, + .controlVersion = pgInterfaceControlVersion96_10, + .controlIs = pgInterfaceControlIs96, .controlCrcOffset = pgInterfaceControlCrcOffset96_10, }, @@ -170,12 +200,12 @@ def test_postgres_render_interface(): interface, ) - # A version that shares every one of its functions has nothing to render, so it gets no block at all + # A version that shares every one of its functions still gets a block, since it captures its own values there interface, _ = _render("version:\n - 9.6\n - 10\n", intern=INTERN_SHARE) - assert_in("PostgreSQL 9.6 interface\n", interface) - assert_not_in("PostgreSQL 10 interface", interface) assert_in("PG_INTERFACE_CONTROL_CRC_OFFSET(96_10); // Shared with 10\n", interface) + assert_not_in("PG_INTERFACE_CONTROL_CRC_OFFSET(10)", interface) + assert_in("PostgreSQL 10 interface\n", interface) #################################################################################################################################### @@ -195,7 +225,14 @@ def test_postgres_render_interface_harness(): assert_in("HRN_PG_INTERFACE_WAL(96_10); // Shared with 10\n", interface) assert_in("#undef HRN_PG_INTERFACE_CONTROL\n", interface) - # The struct and its functions carry the harness prefix + # This interface does not declare the vendored types, it renames the ones the interface it shims declared, so it neither says + # which version they are declared for nor undefines what declaring them defined + assert_not_in("#define PG_VERSION ", interface) + assert_not_in("#define CATALOG_VERSION_NO_MAX\n", interface) + assert_not_in("#undef CATALOG_VERSION_NO\n", interface) + assert_in("#undef TransactionId\n", interface) + + # The struct and its functions carry the harness prefix, and it carries no values since it reads them from what it shims assert_in( """static const HrnPgInterface hrnPgInterface[] = { @@ -216,6 +253,9 @@ def test_postgres_render_interface_harness(): interface, ) + # The harness declares no match, since a test asks for the version it wants rather than matching one + assert_not_in("match", interface) + #################################################################################################################################### def test_postgres_render_interface_unreleased(): @@ -234,6 +274,14 @@ def test_postgres_render_interface_unreleased(): assert_in("PG_INTERFACE_CONTROL_IS(10);\n", interface) assert_in("PG_INTERFACE_CONTROL_CRC_OFFSET(10_19); // Shared with 19\n", interface) + # A version that has not been released accepts a range of values, so the match that compares a range is the one rendered + assert_in("PG_INTERFACE_CONTROL_MATCH_RANGE();\n", interface) + assert_not_in("PG_INTERFACE_CONTROL_MATCH();", interface) + + # Only that version is marked, since the declaration is what says so rather than anything the interface header captures + assert_in(" .version = PG_VERSION_19,\n .unreleased = true,\n", interface) + assert_equal(interface.count(".unreleased = true,"), 1) + #################################################################################################################################### def test_postgres_render_interface_share_error(): diff --git a/test/src/module/postgres/interfaceTest.c b/test/src/module/postgres/interfaceTest.c index c300de674..5f8ceb41e 100644 --- a/test/src/module/postgres/interfaceTest.c +++ b/test/src/module/postgres/interfaceTest.c @@ -276,7 +276,7 @@ testRun(void) // Start with an invalid crc but write a valid one at a greater offset to test the forced scan Buffer *control = hrnPgControlToBuffer(0, 0xFADEFADE, (PgControl){.version = PG_VERSION_13, .systemId = 0xAAAA0AAAA}); - size_t crcOffset = pgInterfaceVersion(PG_VERSION_13)->controlCrcOffset() + sizeof(uint32_t) * 4; + size_t crcOffset = pgInterfaceVersion(PG_VERSION_13)->controlCrcOffset + sizeof(uint32_t) * 4; *((uint32_t *)(bufPtr(control) + crcOffset)) = crc32cOne(bufPtrConst(control), crcOffset); HRN_STORAGE_PUT(storageTest, PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, control); @@ -304,7 +304,7 @@ testRun(void) // Start with an invalid crc but write a valid one at a greater offset to test the forced scan control = hrnPgControlToBuffer( 0, 0xFADEFADE, (PgControl){.version = PG_VERSION_18, .systemId = 0xAAAA0AAAA, .checkpoint = 777}); - crcOffset = pgInterfaceVersion(PG_VERSION_18)->controlCrcOffset() + sizeof(uint32_t) * 2; + crcOffset = pgInterfaceVersion(PG_VERSION_18)->controlCrcOffset + sizeof(uint32_t) * 2; *((uint32_t *)(bufPtr(control) + crcOffset)) = crc32cOne(bufPtrConst(control), crcOffset); info = pgControlFromBuffer(control, STRDEF(PG_VERSION_18_Z));