database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-09-02 04:21:28 +03:00
Code Activity

Handle extension members when first setting object dump flags in pg_dump.

Browse Source 
pg_dump's original approach to handling extension member objects was to
run around and clear (or set) their dump flags rather late in its data
collection process.  Unfortunately, quite a lot of code expects those flags
to be valid before that; which was an entirely reasonable expectation
before we added extensions.  In particular, this explains Karsten Hilbert's
recent report of pg_upgrade failing on a database in which an extension
has been installed into the pg_catalog schema.  Its objects are initially
marked as not-to-be-dumped on the strength of their schema, and later we
change them to must-dump because we're doing a binary upgrade of their
extension; but we've already skipped essential tasks like making associated
DO_SHELL_TYPE objects.

To fix, collect extension membership data first, and incorporate it in the
initial setting of the dump flags, so that those are once again correct
from the get-go.  This has the undesirable side effect of slightly
lengthening the time taken before pg_dump acquires table locks, but testing
suggests that the increase in that window is not very much.

Along the way, get rid of ugly special-case logic for deciding whether
to dump procedural languages, FDWs, and foreign servers; dump decisions
for those are now correct up-front, too.

In 9.3 and up, this also fixes erroneous logic about when to dump event
triggers (basically, they were *always* dumped before).  In 9.5 and up,
transform objects had that problem too.

Since this problem came in with extensions, back-patch to all supported
versions.
This commit is contained in:
Tom Lane
2016-01-13 18:55:27 -05:00
parent 3843ba510b
commit be2b276514
3 changed files with 314 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

310
src/bin/pg_dump/pg_dump.c
View File

@@ -1086,6 +1086,44 @@ expand_table_name_patterns(Archive *fout,
destroyPQExpBuffer(query);
}
/*
* checkExtensionMembership
* Determine whether object is an extension member, and if so,
* record an appropriate dependency and set the object's dump flag.
*
* It's important to call this for each object that could be an extension
* member. Generally, we integrate this with determining the object's
* to-be-dumped-ness, since extension membership overrides other rules for that.
*
* Returns true if object is an extension member, else false.
*/
static bool
checkExtensionMembership(DumpableObject *dobj)
{
ExtensionInfo *ext = findOwningExtension(dobj->catId);
if (ext == NULL)
return false;
dobj->ext_member = true;
/* Record dependency so that getDependencies needn't deal with that */
addObjectDependency(dobj, ext->dobj.dumpId);
/*
* Normally, mark the member object as not to be dumped. But in binary
* upgrades, we still dump the members individually, since the idea is to
* exactly reproduce the database contents rather than replace the
* extension contents with something different.
*/
if (!binary_upgrade)
dobj->dump = false;
else
dobj->dump = ext->dobj.dump;
return true;
}
/*
* selectDumpableNamespace: policy-setting subroutine
* Mark a namespace as to be dumped or not
@@ -1093,6 +1131,9 @@ expand_table_name_patterns(Archive *fout,
static void
selectDumpableNamespace(NamespaceInfo *nsinfo)
{
if (checkExtensionMembership(&nsinfo->dobj))
return; /* extension membership overrides all else */
/*
* If specific tables are being dumped, do not dump any complete
* namespaces. If specific namespaces are being dumped, dump just those
@@ -1125,6 +1166,9 @@ selectDumpableNamespace(NamespaceInfo *nsinfo)
static void
selectDumpableTable(TableInfo *tbinfo)
{
if (checkExtensionMembership(&tbinfo->dobj))
return; /* extension membership overrides all else */
/*
* If specific tables are being dumped, dump just those tables; else, dump
* according to the parent namespace's dump flag.
@@ -1187,6 +1231,9 @@ selectDumpableType(TypeInfo *tyinfo)
*/
}
if (checkExtensionMembership(&tyinfo->dobj))
return; /* extension membership overrides all else */
/* dump only types in dumpable namespaces */
if (!tyinfo->dobj.namespace->dobj.dump)
tyinfo->dobj.dump = false;
@@ -1205,6 +1252,8 @@ selectDumpableType(TypeInfo *tyinfo)
static void
selectDumpableDefaultACL(DefaultACLInfo *dinfo)
{
/* Default ACLs can't be extension members */
if (dinfo->dobj.namespace)
dinfo->dobj.dump = dinfo->dobj.namespace->dobj.dump;
else
@@ -1223,12 +1272,35 @@ selectDumpableDefaultACL(DefaultACLInfo *dinfo)
static void
selectDumpableCast(CastInfo *cast)
{
if (checkExtensionMembership(&cast->dobj))
return; /* extension membership overrides all else */
if (cast->dobj.catId.oid < (Oid) FirstNormalObjectId)
cast->dobj.dump = false;
else
cast->dobj.dump = include_everything;
}
/*
* selectDumpableProcLang: policy-setting subroutine
* Mark a procedural language as to be dumped or not
*
* Procedural languages do not belong to any particular namespace. To
* identify built-in languages, we must resort to checking whether the
* language's OID is in the range reserved for initdb.
*/
static void
selectDumpableProcLang(ProcLangInfo *plang)
{
if (checkExtensionMembership(&plang->dobj))
return; /* extension membership overrides all else */
if (plang->dobj.catId.oid < (Oid) FirstNormalObjectId)
plang->dobj.dump = false;
else
plang->dobj.dump = include_everything;
}
/*
* selectDumpableExtension: policy-setting subroutine
* Mark an extension as to be dumped or not
@@ -1257,14 +1329,17 @@ selectDumpableExtension(ExtensionInfo *extinfo)
static void
selectDumpableObject(DumpableObject *dobj)
{
if (checkExtensionMembership(dobj))
return; /* extension membership overrides all else */
/*
* Default policy is to dump if parent namespace is dumpable, or always
* for non-namespace-associated items.
* Default policy is to dump if parent namespace is dumpable, or for
* non-namespace-associated items, dump if we're dumping "everything".
*/
if (dobj->namespace)
dobj->dump = dobj->namespace->dobj.dump;
else
dobj->dump = true;
dobj->dump = include_everything;
}
/*
@@ -5517,6 +5592,9 @@ getProcLangs(Archive *fout, int *numProcLangs)
planginfo[i].lanacl = pg_strdup(PQgetvalue(res, i, i_lanacl));
planginfo[i].lanowner = pg_strdup(PQgetvalue(res, i, i_lanowner));
/* Decide whether we want to dump it */
selectDumpableProcLang(&(planginfo[i]));
if (fout->remoteVersion < 70300)
{
/*
@@ -8782,32 +8860,6 @@ dumpShellType(Archive *fout, ShellTypeInfo *stinfo)
destroyPQExpBuffer(q);
}
/*
* Determine whether we want to dump definitions for procedural languages.
* Since the languages themselves don't have schemas, we can't rely on
* the normal schema-based selection mechanism. We choose to dump them
* whenever neither --schema nor --table was given. (Before 8.1, we used
* the dump flag of the PL's call handler function, but in 8.1 this will
* probably always be false since call handlers are created in pg_catalog.)
*
* For some backwards compatibility with the older behavior, we forcibly
* dump a PL if its handler function (and validator if any) are in a
* dumpable namespace. That case is not checked here.
*
* Also, if the PL belongs to an extension, we do not use this heuristic.
* That case isn't checked here either.
*/
static bool
shouldDumpProcLangs(void)
{
if (!include_everything)
return false;
/* And they're schema not data */
if (dataOnly)
return false;
return true;
}
/*
* dumpProcLang
* writes out to fout the queries to recreate a user-defined
@@ -8858,25 +8910,13 @@ dumpProcLang(Archive *fout, ProcLangInfo *plang)
/*
* If the functions are dumpable then emit a traditional CREATE LANGUAGE
* with parameters. Otherwise, dump only if shouldDumpProcLangs() says to
* dump it.
*
* However, for a language that belongs to an extension, we must not use
* the shouldDumpProcLangs heuristic, but just dump the language iff we're
* told to (via dobj.dump). Generally the support functions will belong
* to the same extension and so have the same dump flags ... if they
* don't, this might not work terribly nicely.
* with parameters. Otherwise, we'll write a parameterless command, which
* will rely on data from pg_pltemplate.
*/
useParams = (funcInfo != NULL &&
(inlineInfo != NULL || !OidIsValid(plang->laninline)) &&
(validatorInfo != NULL || !OidIsValid(plang->lanvalidator)));
if (!plang->dobj.ext_member)
{
if (!useParams && !shouldDumpProcLangs())
return;
}
defqry = createPQExpBuffer();
delqry = createPQExpBuffer();
labelq = createPQExpBuffer();
@@ -11549,14 +11589,6 @@ dumpForeignDataWrapper(Archive *fout, FdwInfo *fdwinfo)
if (!fdwinfo->dobj.dump || dataOnly)
return;
/*
* FDWs that belong to an extension are dumped based on their "dump"
* field. Otherwise omit them if we are only dumping some specific object.
*/
if (!fdwinfo->dobj.ext_member)
if (!include_everything)
return;
q = createPQExpBuffer();
delq = createPQExpBuffer();
labelq = createPQExpBuffer();
@@ -11631,7 +11663,7 @@ dumpForeignServer(Archive *fout, ForeignServerInfo *srvinfo)
char *fdwname;
/* Skip if not to be dumped */
if (!srvinfo->dobj.dump || dataOnly || !include_everything)
if (!srvinfo->dobj.dump || dataOnly)
return;
q = createPQExpBuffer();
@@ -13979,32 +14011,12 @@ dumpRule(Archive *fout, RuleInfo *rinfo)
/*
* getExtensionMembership --- obtain extension membership data
*
* There are three main parts to this process:
*
* 1. Identify objects which are members of extensions
*
* Generally speaking, this is to mark them as *not* being dumped, as most
* extension objects are created by the single CREATE EXTENSION command.
* The one exception is binary upgrades with pg_upgrade will still dump the
* non-table objects.
*
* 2. Identify and create dump records for extension configuration tables.
*
* Extensions can mark tables as "configuration", which means that the user
* is able and expected to modify those tables after the extension has been
* loaded. For these tables, we dump out only the data- the structure is
* expected to be handled at CREATE EXTENSION time, including any indexes or
* foriegn keys, which brings us to-
*
* 3. Record FK dependencies between configuration tables.
*
* Due to the FKs being created at CREATE EXTENSION time and therefore before
* the data is loaded, we have to work out what the best order for reloading
* the data is, to avoid FK violations when the tables are restored. This is
* not perfect- we can't handle circular dependencies and if any exist they
* will cause an invalid dump to be produced (though at least all of the data
* is included for a user to manually restore). This is currently documented
* but perhaps we can provide a better solution in the future.
* We need to identify objects that are extension members as soon as they're
* loaded, so that we can correctly determine whether they need to be dumped.
* Generally speaking, extension member objects will get marked as *not* to
* be dumped, as they will be recreated by the single CREATE EXTENSION
* command. However, in binary upgrade mode we still need to dump the members
* individually.
*/
void
getExtensionMembership(Archive *fout, ExtensionInfo extinfo[],
@@ -14013,15 +14025,13 @@ getExtensionMembership(Archive *fout, ExtensionInfo extinfo[],
PQExpBuffer query;
PGresult *res;
int ntups,
nextmembers,
i;
int i_classid,
i_objid,
i_refclassid,
i_refobjid,
i_conrelid,
i_confrelid;
DumpableObject *dobj,
*refdobj;
i_refobjid;
ExtensionMemberId *extmembers;
ExtensionInfo *ext;
/* Nothing to do if no extensions */
if (numExtensions == 0)
@@ -14033,12 +14043,12 @@ getExtensionMembership(Archive *fout, ExtensionInfo extinfo[],
query = createPQExpBuffer();
/* refclassid constraint is redundant but may speed the search */
appendPQExpBuffer(query, "SELECT "
"classid, objid, refclassid, refobjid "
"FROM pg_depend "
"WHERE refclassid = 'pg_extension'::regclass "
"AND deptype = 'e' "
"ORDER BY 3,4");
appendPQExpBufferStr(query, "SELECT "
"classid, objid, refobjid "
"FROM pg_depend "
"WHERE refclassid = 'pg_extension'::regclass "
"AND deptype = 'e' "
"ORDER BY 3");
res = ExecuteSqlQuery(fout, query->data, PGRES_TUPLES_OK);
@@ -14046,76 +14056,93 @@ getExtensionMembership(Archive *fout, ExtensionInfo extinfo[],
i_classid = PQfnumber(res, "classid");
i_objid = PQfnumber(res, "objid");
i_refclassid = PQfnumber(res, "refclassid");
i_refobjid = PQfnumber(res, "refobjid");
extmembers = (ExtensionMemberId *) pg_malloc(ntups * sizeof(ExtensionMemberId));
nextmembers = 0;
/*
* Accumulate data into extmembers[].
*
* Since we ordered the SELECT by referenced ID, we can expect that
* multiple entries for the same extension will appear together; this
* saves on searches.
*/
refdobj = NULL;
ext = NULL;
for (i = 0; i < ntups; i++)
{
CatalogId objId;
CatalogId refobjId;
Oid extId;
objId.tableoid = atooid(PQgetvalue(res, i, i_classid));
objId.oid = atooid(PQgetvalue(res, i, i_objid));
refobjId.tableoid = atooid(PQgetvalue(res, i, i_refclassid));
refobjId.oid = atooid(PQgetvalue(res, i, i_refobjid));
extId = atooid(PQgetvalue(res, i, i_refobjid));
if (refdobj == NULL ||
refdobj->catId.tableoid != refobjId.tableoid ||
refdobj->catId.oid != refobjId.oid)
refdobj = findObjectByCatalogId(refobjId);
if (ext == NULL ||
ext->dobj.catId.oid != extId)
ext = findExtensionByOid(extId);
/*
* Failure to find objects mentioned in pg_depend is not unexpected,
* since for example we don't collect info about TOAST tables.
*/
if (refdobj == NULL)
if (ext == NULL)
{
#ifdef NOT_USED
fprintf(stderr, "no referenced object %u %u\n",
refobjId.tableoid, refobjId.oid);
#endif
/* shouldn't happen */
fprintf(stderr, "could not find referenced extension %u\n", extId);
continue;
}
dobj = findObjectByCatalogId(objId);
if (dobj == NULL)
{
#ifdef NOT_USED
fprintf(stderr, "no referencing object %u %u\n",
objId.tableoid, objId.oid);
#endif
continue;
}
/* Record dependency so that getDependencies needn't repeat this */
addObjectDependency(dobj, refdobj->dumpId);
dobj->ext_member = true;
/*
* Normally, mark the member object as not to be dumped. But in
* binary upgrades, we still dump the members individually, since the
* idea is to exactly reproduce the database contents rather than
* replace the extension contents with something different.
*/
if (!binary_upgrade)
dobj->dump = false;
else
dobj->dump = refdobj->dump;
extmembers[nextmembers].catId = objId;
extmembers[nextmembers].ext = ext;
nextmembers++;
}
PQclear(res);
/* Remember the data for use later */
setExtensionMembership(extmembers, nextmembers);
destroyPQExpBuffer(query);
}
/*
* processExtensionTables --- deal with extension configuration tables
*
* There are two parts to this process:
*
* 1. Identify and create dump records for extension configuration tables.
*
* Extensions can mark tables as "configuration", which means that the user
* is able and expected to modify those tables after the extension has been
* loaded. For these tables, we dump out only the data- the structure is
* expected to be handled at CREATE EXTENSION time, including any indexes or
* foreign keys, which brings us to-
*
* 2. Record FK dependencies between configuration tables.
*
* Due to the FKs being created at CREATE EXTENSION time and therefore before
* the data is loaded, we have to work out what the best order for reloading
* the data is, to avoid FK violations when the tables are restored. This is
* not perfect- we can't handle circular dependencies and if any exist they
* will cause an invalid dump to be produced (though at least all of the data
* is included for a user to manually restore). This is currently documented
* but perhaps we can provide a better solution in the future.
*/
void
processExtensionTables(Archive *fout, ExtensionInfo extinfo[],
int numExtensions)
{
PQExpBuffer query;
PGresult *res;
int ntups,
i;
int i_conrelid,
i_confrelid;
/* Nothing to do if no extensions */
if (numExtensions == 0)
return;
/*
* Now identify extension configuration tables and create TableDataInfo
* Identify extension configuration tables and create TableDataInfo
* objects for them, ensuring their data will be dumped even though the
* tables themselves won't be.
*
@@ -14201,14 +14228,19 @@ getExtensionMembership(Archive *fout, ExtensionInfo extinfo[],
}
/*
* Now that all the TableInfoData objects have been created for all
* the extensions, check their FK dependencies and register them to
* try and dump the data out in an order which they can be restored
* in.
* Now that all the TableInfoData objects have been created for all the
* extensions, check their FK dependencies and register them to try and
* dump the data out in an order that they can be restored in.
*
* Note that this is not a problem for user tables as their FKs are
* recreated after the data has been loaded.
*/
/* Make sure we are in proper schema */
selectSourceSchema(fout, "pg_catalog");
query = createPQExpBuffer();
printfPQExpBuffer(query,
"SELECT conrelid, confrelid "
"FROM pg_constraint "