diff --git a/doc/src/sgml/ref/pg_dump.sgml b/doc/src/sgml/ref/pg_dump.sgml index 8695571045b..0e5ba4f7125 100644 --- a/doc/src/sgml/ref/pg_dump.sgml +++ b/doc/src/sgml/ref/pg_dump.sgml @@ -836,6 +836,109 @@ PostgreSQL documentation + + + + + Specify a filename from which to read patterns for objects to include + or exclude from the dump. The patterns are interpreted according to the + same rules as the corresponding options: + /, + , + or + for tables, + / for schemas, + for data on foreign servers and + , + for table data, + / for extensions. + To read from STDIN, use - as the + filename. The option can be specified in + conjunction with the above listed options for including or excluding + objects, and can also be specified more than once for multiple filter + files. + + + + The file lists one object pattern per row, with the following format: + +{ include | exclude } { extension | foreign_data | table | table_and_children | table_data | table_data_and_children | schema } PATTERN + + + + + The first keyword specifies whether the objects matched by the pattern + are to be included or excluded. The second keyword specifies the type + of object to be filtered using the pattern: + + + + extension: extensions, works like the + option. This keyword can only be + used with the include keyword. + + + + + foreign_data: data on foreign servers, works like + the option. This keyword can + only be used with the include keyword. + + + + + table: tables, works like the + / option. + + + + + table_and_children: tables including any partitions + or inheritance child tables, works like the + option. + + + + + table_data: table data of any tables matching + pattern, works like the + option. This keyword can only + be used with the exclude keyword. + + + + + table_data_and_children: table data of any tables + matching pattern as well as any partitions + or inheritance children of the table(s), works like the + option. This + keyword can only be used with the exclude keyword. + + + + + schema: schemas, works like the + / option. + + + + + + + Lines starting with # are considered comments and + ignored. Comments can be placed after an object pattern row as well. + Blank lines are also ignored. See + for how to perform quoting in patterns. + + + + Example files are listed below in the + section. + + + + + @@ -1168,6 +1271,7 @@ PostgreSQL documentation schema (/) and table (/) pattern match at least one extension/schema/table in the database to be dumped. + This also applies to filters used with . Note that if none of the extension/schema/table patterns find matches, pg_dump will generate an error even without . @@ -1611,6 +1715,19 @@ CREATE DATABASE foo WITH TEMPLATE template0; $ pg_dump -t "\"MixedCaseName\"" mydb > mytab.sql + + + + To dump all tables whose names start with mytable, except + for table mytable2, specify a filter file + filter.txt like: + +include table mytable* +exclude table mytable2 + + + +$ pg_dump --filter=filter.txt mydb > db.sql diff --git a/doc/src/sgml/ref/pg_dumpall.sgml b/doc/src/sgml/ref/pg_dumpall.sgml index d31585216c6..4d7c0464687 100644 --- a/doc/src/sgml/ref/pg_dumpall.sgml +++ b/doc/src/sgml/ref/pg_dumpall.sgml @@ -125,6 +125,37 @@ PostgreSQL documentation + + + + + Specify a filename from which to read patterns for databases excluded + from the dump. The patterns are interpreted according to the same rules + as . + To read from STDIN, use - as the + filename. The option can be specified in + conjunction with for excluding + databases, and can also be specified more than once for multiple filter + files. + + + + The file lists one database pattern per row, with the following format: + +exclude database PATTERN + + + + + Lines starting with # are considered comments and + ignored. Comments can be placed after an object pattern row as well. + Blank lines are also ignored. See + for how to perform quoting in patterns. + + + + + diff --git a/doc/src/sgml/ref/pg_restore.sgml b/doc/src/sgml/ref/pg_restore.sgml index 374d8d8715c..1a23874da68 100644 --- a/doc/src/sgml/ref/pg_restore.sgml +++ b/doc/src/sgml/ref/pg_restore.sgml @@ -190,6 +190,86 @@ PostgreSQL documentation + + + + + Specify a filename from which to read patterns for objects excluded + or included from restore. The patterns are interpreted according to the + same rules as + / for including objects in schemas, + /for excluding objects in schemas, + / for restoring named functions, + / for restoring named indexes, + / for restoring named tables + or / for restoring triggers. + To read from STDIN, use - as the + filename. The option can be specified in + conjunction with the above listed options for including or excluding + objects, and can also be specified more than once for multiple filter + files. + + + + The file lists one database pattern per row, with the following format: + +{ include | exclude } { function | index | schema | table | trigger } PATTERN + + + + + The first keyword specifies whether the objects matched by the pattern + are to be included or excluded. The second keyword specifies the type + of object to be filtered using the pattern: + + + + function: functions, works like the + / option. This keyword + can only be used with the include keyword. + + + + + index: indexes, works like the + / option. This keyword + can only be used with the include keyword. + + + + + schema: schemas, works like the + / and + / options. + + + + + table: tables, works like the + / option. This keyword + can only be used with the include keyword. + + + + + trigger: triggers, works like the + / option. This keyword + can only be used with the include keyword. + + + + + + + Lines starting with # are considered comments and + ignored. Comments can be placed after an object pattern row as well. + Blank lines are also ignored. See + for how to perform quoting in patterns. + + + + + diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile index 604cddb9972..2bcf2a70028 100644 --- a/src/bin/pg_dump/Makefile +++ b/src/bin/pg_dump/Makefile @@ -32,6 +32,7 @@ OBJS = \ compress_none.o \ compress_zstd.o \ dumputils.o \ + filter.o \ parallel.o \ pg_backup_archiver.o \ pg_backup_custom.o \ @@ -49,8 +50,8 @@ pg_dump: pg_dump.o common.o pg_dump_sort.o $(OBJS) | submake-libpq submake-libpg pg_restore: pg_restore.o $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils $(CC) $(CFLAGS) pg_restore.o $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) -pg_dumpall: pg_dumpall.o dumputils.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils - $(CC) $(CFLAGS) pg_dumpall.o dumputils.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) +pg_dumpall: pg_dumpall.o dumputils.o filter.o $(WIN32RES) | submake-libpq submake-libpgport submake-libpgfeutils + $(CC) $(CFLAGS) pg_dumpall.o dumputils.o filter.o $(WIN32RES) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) install: all installdirs $(INSTALL_PROGRAM) pg_dump$(X) '$(DESTDIR)$(bindir)'/pg_dump$(X) diff --git a/src/bin/pg_dump/filter.c b/src/bin/pg_dump/filter.c new file mode 100644 index 00000000000..dff871c7cd0 --- /dev/null +++ b/src/bin/pg_dump/filter.c @@ -0,0 +1,471 @@ +/*------------------------------------------------------------------------- + * + * filter.c + * Implementation of simple filter file parser + * + * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/bin/pg_dump/filter.c + * + *------------------------------------------------------------------------- + */ +#include "postgres_fe.h" + +#include "common/fe_memutils.h" +#include "common/logging.h" +#include "common/string.h" +#include "filter.h" +#include "lib/stringinfo.h" +#include "pqexpbuffer.h" + +#define is_keyword_str(cstr, str, bytes) \ + ((strlen(cstr) == (bytes)) && (pg_strncasecmp((cstr), (str), (bytes)) == 0)) + +/* + * Following routines are called from pg_dump, pg_dumpall and pg_restore. + * Since the implementation of exit_nicely is application specific, each + * application need to pass a function pointer to the exit_nicely function to + * use for exiting on errors. + */ + +/* + * Opens filter's file and initialize fstate structure. + */ +void +filter_init(FilterStateData *fstate, const char *filename, exit_function f_exit) +{ + fstate->filename = filename; + fstate->lineno = 0; + fstate->exit_nicely = f_exit; + initStringInfo(&fstate->linebuff); + + if (strcmp(filename, "-") != 0) + { + fstate->fp = fopen(filename, "r"); + if (!fstate->fp) + { + pg_log_error("could not open filter file \"%s\": %m", filename); + fstate->exit_nicely(1); + } + } + else + fstate->fp = stdin; +} + +/* + * Release allocated resources for the given filter. + */ +void +filter_free(FilterStateData *fstate) +{ + if (!fstate) + return; + + free(fstate->linebuff.data); + fstate->linebuff.data = NULL; + + if (fstate->fp && fstate->fp != stdin) + { + if (fclose(fstate->fp) != 0) + pg_log_error("could not close filter file \"%s\": %m", fstate->filename); + + fstate->fp = NULL; + } +} + +/* + * Translate FilterObjectType enum to string. The main purpose is for error + * message formatting. + */ +const char * +filter_object_type_name(FilterObjectType fot) +{ + switch (fot) + { + case FILTER_OBJECT_TYPE_NONE: + return "comment or empty line"; + case FILTER_OBJECT_TYPE_TABLE_DATA: + return "table data"; + case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN: + return "table data and children"; + case FILTER_OBJECT_TYPE_DATABASE: + return "database"; + case FILTER_OBJECT_TYPE_EXTENSION: + return "extension"; + case FILTER_OBJECT_TYPE_FOREIGN_DATA: + return "foreign data"; + case FILTER_OBJECT_TYPE_FUNCTION: + return "function"; + case FILTER_OBJECT_TYPE_INDEX: + return "index"; + case FILTER_OBJECT_TYPE_SCHEMA: + return "schema"; + case FILTER_OBJECT_TYPE_TABLE: + return "table"; + case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN: + return "table and children"; + case FILTER_OBJECT_TYPE_TRIGGER: + return "trigger"; + } + + /* should never get here */ + pg_unreachable(); +} + +/* + * Returns true when keyword is one of supported object types, and + * set related objtype. Returns false, when keyword is not assigned + * with known object type. + */ +static bool +get_object_type(const char *keyword, int size, FilterObjectType *objtype) +{ + if (is_keyword_str("table_data", keyword, size)) + *objtype = FILTER_OBJECT_TYPE_TABLE_DATA; + else if (is_keyword_str("table_data_and_children", keyword, size)) + *objtype = FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN; + else if (is_keyword_str("database", keyword, size)) + *objtype = FILTER_OBJECT_TYPE_DATABASE; + else if (is_keyword_str("extension", keyword, size)) + *objtype = FILTER_OBJECT_TYPE_EXTENSION; + else if (is_keyword_str("foreign_data", keyword, size)) + *objtype = FILTER_OBJECT_TYPE_FOREIGN_DATA; + else if (is_keyword_str("function", keyword, size)) + *objtype = FILTER_OBJECT_TYPE_FUNCTION; + else if (is_keyword_str("index", keyword, size)) + *objtype = FILTER_OBJECT_TYPE_INDEX; + else if (is_keyword_str("schema", keyword, size)) + *objtype = FILTER_OBJECT_TYPE_SCHEMA; + else if (is_keyword_str("table", keyword, size)) + *objtype = FILTER_OBJECT_TYPE_TABLE; + else if (is_keyword_str("table_and_children", keyword, size)) + *objtype = FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN; + else if (is_keyword_str("trigger", keyword, size)) + *objtype = FILTER_OBJECT_TYPE_TRIGGER; + else + return false; + + return true; +} + + +void +pg_log_filter_error(FilterStateData *fstate, const char *fmt,...) +{ + va_list argp; + char buf[256]; + + va_start(argp, fmt); + vsnprintf(buf, sizeof(buf), fmt, argp); + va_end(argp); + + pg_log_error("invalid format in filter read from \"%s\" on line %d: %s", + (fstate->fp == stdin ? "stdin" : fstate->filename), + fstate->lineno, + buf); +} + +/* + * filter_get_keyword - read the next filter keyword from buffer + * + * Search for keywords (limited to ascii alphabetic characters) in + * the passed in line buffer. Returns NULL when the buffer is empty or the first + * char is not alpha. The char '_' is allowed, except as the first character. + * The length of the found keyword is returned in the size parameter. + */ +static const char * +filter_get_keyword(const char **line, int *size) +{ + const char *ptr = *line; + const char *result = NULL; + + /* Set returned length preemptively in case no keyword is found */ + *size = 0; + + /* Skip initial whitespace */ + while (isspace(*ptr)) + ptr++; + + if (isalpha(*ptr)) + { + result = ptr++; + + while (isalpha(*ptr) || *ptr == '_') + ptr++; + + *size = ptr - result; + } + + *line = ptr; + + return result; +} + +/* + * read_quoted_pattern - read quoted possibly multi line string + * + * Reads a quoted string which can span over multiple lines and returns a + * pointer to next char after ending double quotes; it will exit on errors. + */ +static const char * +read_quoted_string(FilterStateData *fstate, + const char *str, + PQExpBuffer pattern) +{ + appendPQExpBufferChar(pattern, '"'); + str++; + + while (1) + { + /* + * We can ignore \r or \n chars because the string is read by + * pg_get_line_buf, so these chars should be just trailing chars. + */ + if (*str == '\r' || *str == '\n') + { + str++; + continue; + } + + if (*str == '\0') + { + Assert(fstate->linebuff.data); + + if (!pg_get_line_buf(fstate->fp, &fstate->linebuff)) + { + if (ferror(fstate->fp)) + pg_log_error("could not read from filter file \"%s\": %m", + fstate->filename); + else + pg_log_filter_error(fstate, _("unexpected end of file")); + + fstate->exit_nicely(1); + } + + str = fstate->linebuff.data; + + appendPQExpBufferChar(pattern, '\n'); + fstate->lineno++; + } + + if (*str == '"') + { + appendPQExpBufferChar(pattern, '"'); + str++; + + if (*str == '"') + { + appendPQExpBufferChar(pattern, '"'); + str++; + } + else + break; + } + else if (*str == '\\') + { + str++; + if (*str == 'n') + appendPQExpBufferChar(pattern, '\n'); + else if (*str == '\\') + appendPQExpBufferChar(pattern, '\\'); + + str++; + } + else + appendPQExpBufferChar(pattern, *str++); + } + + return str; +} + +/* + * read_pattern - reads on object pattern from input + * + * This function will parse any valid identifier (quoted or not, qualified or + * not), which can also includes the full signature for routines. + * Note that this function takes special care to sanitize the detected + * identifier (removing extraneous whitespaces or other unnecessary + * characters). This is necessary as most backup/restore filtering functions + * only recognize identifiers if they are written exactly the same way as + * they are output by the server. + * + * Returns a pointer to next character after the found identifier and exits + * on error. + */ +static const char * +read_pattern(FilterStateData *fstate, const char *str, PQExpBuffer pattern) +{ + bool skip_space = true; + bool found_space = false; + + /* Skip initial whitespace */ + while (isspace(*str)) + str++; + + if (*str == '\0') + { + pg_log_filter_error(fstate, _("missing object name pattern")); + fstate->exit_nicely(1); + } + + while (*str && *str != '#') + { + while (*str && !isspace(*str) && !strchr("#,.()\"", *str)) + { + /* + * Append space only when it is allowed, and when it was found in + * original string. + */ + if (!skip_space && found_space) + { + appendPQExpBufferChar(pattern, ' '); + skip_space = true; + } + + appendPQExpBufferChar(pattern, *str++); + } + + skip_space = false; + + if (*str == '"') + { + if (found_space) + appendPQExpBufferChar(pattern, ' '); + + str = read_quoted_string(fstate, str, pattern); + } + else if (*str == ',') + { + appendPQExpBufferStr(pattern, ", "); + skip_space = true; + str++; + } + else if (*str && strchr(".()", *str)) + { + appendPQExpBufferChar(pattern, *str++); + skip_space = true; + } + + found_space = false; + + /* skip ending whitespaces */ + while (isspace(*str)) + { + found_space = true; + str++; + } + } + + return str; +} + +/* + * filter_read_item - Read command/type/pattern triplet from a filter file + * + * This will parse one filter item from the filter file, and while it is a + * row based format a pattern may span more than one line due to how object + * names can be constructed. The expected format of the filter file is: + * + * + * + * command can be "include" or "exclude". + * + * Supported object types are described by enum FilterObjectType + * (see function get_object_type). + * + * pattern can be any possibly-quoted and possibly-qualified identifier. It + * follows the same rules as other object include and exclude functions so it + * can also use wildcards. + * + * Returns true when one filter item was successfully read and parsed. When + * object name contains \n chars, then more than one line from input file can + * be processed. Returns false when the filter file reaches EOF. In case of + * error, the function will emit an appropriate error message and exit. + */ +bool +filter_read_item(FilterStateData *fstate, + char **objname, + FilterCommandType *comtype, + FilterObjectType *objtype) +{ + if (pg_get_line_buf(fstate->fp, &fstate->linebuff)) + { + const char *str = fstate->linebuff.data; + const char *keyword; + int size; + PQExpBufferData pattern; + + fstate->lineno++; + + /* Skip initial white spaces */ + while (isspace(*str)) + str++; + + /* + * Skip empty lines or lines where the first non-whitespace character + * is a hash indicating a comment. + */ + if (*str != '\0' && *str != '#') + { + /* + * First we expect sequence of two keywords, {include|exclude} + * followed by the object type to operate on. + */ + keyword = filter_get_keyword(&str, &size); + if (!keyword) + { + pg_log_filter_error(fstate, + _("no filter command found (expected \"include\" or \"exclude\")")); + fstate->exit_nicely(1); + } + + if (is_keyword_str("include", keyword, size)) + *comtype = FILTER_COMMAND_TYPE_INCLUDE; + else if (is_keyword_str("exclude", keyword, size)) + *comtype = FILTER_COMMAND_TYPE_EXCLUDE; + else + { + pg_log_filter_error(fstate, + _("invalid filter command (expected \"include\" or \"exclude\")")); + fstate->exit_nicely(1); + } + + keyword = filter_get_keyword(&str, &size); + if (!keyword) + { + pg_log_filter_error(fstate, _("missing filter object type")); + fstate->exit_nicely(1); + } + + if (!get_object_type(keyword, size, objtype)) + { + pg_log_filter_error(fstate, + _("unsupported filter object type: \"%.*s\""), size, keyword); + fstate->exit_nicely(1); + } + + initPQExpBuffer(&pattern); + + str = read_pattern(fstate, str, &pattern); + *objname = pattern.data; + } + else + { + *objname = NULL; + *comtype = FILTER_COMMAND_TYPE_NONE; + *objtype = FILTER_OBJECT_TYPE_NONE; + } + + return true; + } + + if (ferror(fstate->fp)) + { + pg_log_error("could not read from filter file \"%s\": %m", fstate->filename); + fstate->exit_nicely(1); + } + + return false; +} diff --git a/src/bin/pg_dump/filter.h b/src/bin/pg_dump/filter.h new file mode 100644 index 00000000000..502407d4fbc --- /dev/null +++ b/src/bin/pg_dump/filter.h @@ -0,0 +1,71 @@ +/*------------------------------------------------------------------------- + * + * filter.h + * Common header file for the parser of filter file + * + * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/bin/pg_dump/filter.h + * + *------------------------------------------------------------------------- + */ +#ifndef FILTER_H +#define FILTER_H + +#include "lib/stringinfo.h" + +/* Function signature for exit_nicely functions */ +typedef void (*exit_function) (int status); + +/* + * State data for reading filter items from stream + */ +typedef struct +{ + FILE *fp; + const char *filename; + exit_function exit_nicely; + int lineno; + StringInfoData linebuff; +} FilterStateData; + +/* + * List of command types that can be specified in filter file + */ +typedef enum +{ + FILTER_COMMAND_TYPE_NONE, + FILTER_COMMAND_TYPE_INCLUDE, + FILTER_COMMAND_TYPE_EXCLUDE, +} FilterCommandType; + +/* + * List of objects that can be specified in filter file + */ +typedef enum +{ + FILTER_OBJECT_TYPE_NONE, + FILTER_OBJECT_TYPE_TABLE_DATA, + FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN, + FILTER_OBJECT_TYPE_DATABASE, + FILTER_OBJECT_TYPE_EXTENSION, + FILTER_OBJECT_TYPE_FOREIGN_DATA, + FILTER_OBJECT_TYPE_FUNCTION, + FILTER_OBJECT_TYPE_INDEX, + FILTER_OBJECT_TYPE_SCHEMA, + FILTER_OBJECT_TYPE_TABLE, + FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN, + FILTER_OBJECT_TYPE_TRIGGER, +} FilterObjectType; + +extern const char *filter_object_type_name(FilterObjectType fot); +extern void filter_init(FilterStateData *fstate, const char *filename, exit_function f_exit); +extern void filter_free(FilterStateData *fstate); +extern void pg_log_filter_error(FilterStateData *fstate, const char *fmt,...) + pg_attribute_printf(2, 3); +extern bool filter_read_item(FilterStateData *fstate, char **objname, + FilterCommandType *comtype, FilterObjectType *objtype); + +#endif /* FILTER_H */ diff --git a/src/bin/pg_dump/meson.build b/src/bin/pg_dump/meson.build index 9d59a106f36..b6603e26a50 100644 --- a/src/bin/pg_dump/meson.build +++ b/src/bin/pg_dump/meson.build @@ -7,6 +7,7 @@ pg_dump_common_sources = files( 'compress_none.c', 'compress_zstd.c', 'dumputils.c', + 'filter.c', 'parallel.c', 'pg_backup_archiver.c', 'pg_backup_custom.c', @@ -99,6 +100,7 @@ tests += { 't/002_pg_dump.pl', 't/003_pg_dump_with_server.pl', 't/004_pg_dump_parallel.pl', + 't/005_pg_dump_filterfile.pl', 't/010_dump_connstr.pl', ], }, diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 34fd0a86e9c..64e2d754d12 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -60,6 +60,7 @@ #include "dumputils.h" #include "fe_utils/option_utils.h" #include "fe_utils/string_utils.h" +#include "filter.h" #include "getopt_long.h" #include "libpq/libpq-fs.h" #include "parallel.h" @@ -327,6 +328,7 @@ static char *get_synchronized_snapshot(Archive *fout); static void setupDumpWorker(Archive *AH); static TableInfo *getRootTableInfo(const TableInfo *tbinfo); static bool forcePartitionRootLoad(const TableInfo *tbinfo); +static void read_dump_filters(const char *filename, DumpOptions *dopt); int @@ -433,6 +435,7 @@ main(int argc, char **argv) {"exclude-table-and-children", required_argument, NULL, 13}, {"exclude-table-data-and-children", required_argument, NULL, 14}, {"sync-method", required_argument, NULL, 15}, + {"filter", required_argument, NULL, 16}, {NULL, 0, NULL, 0} }; @@ -664,6 +667,10 @@ main(int argc, char **argv) exit_nicely(1); break; + case 16: /* read object filters from file */ + read_dump_filters(optarg, &dopt); + break; + default: /* getopt_long already emitted a complaint */ pg_log_error_hint("Try \"%s --help\" for more information.", progname); @@ -1111,6 +1118,8 @@ help(const char *progname) " do NOT dump data for the specified table(s),\n" " including child and partition tables\n")); printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n")); + printf(_(" --filter=FILENAME include or exclude objects and data from dump\n" + " based expressions in FILENAME\n")); printf(_(" --if-exists use IF EXISTS when dropping objects\n")); printf(_(" --include-foreign-data=PATTERN\n" " include data of foreign tables on foreign\n" @@ -18771,3 +18780,112 @@ appendReloptionsArrayAH(PQExpBuffer buffer, const char *reloptions, if (!res) pg_log_warning("could not parse %s array", "reloptions"); } + +/* + * read_dump_filters - retrieve object identifier patterns from file + * + * Parse the specified filter file for include and exclude patterns, and add + * them to the relevant lists. If the filename is "-" then filters will be + * read from STDIN rather than a file. + */ +static void +read_dump_filters(const char *filename, DumpOptions *dopt) +{ + FilterStateData fstate; + char *objname; + FilterCommandType comtype; + FilterObjectType objtype; + + filter_init(&fstate, filename, exit_nicely); + + while (filter_read_item(&fstate, &objname, &comtype, &objtype)) + { + if (comtype == FILTER_COMMAND_TYPE_INCLUDE) + { + switch (objtype) + { + case FILTER_OBJECT_TYPE_NONE: + break; + case FILTER_OBJECT_TYPE_DATABASE: + case FILTER_OBJECT_TYPE_FUNCTION: + case FILTER_OBJECT_TYPE_INDEX: + case FILTER_OBJECT_TYPE_TABLE_DATA: + case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN: + case FILTER_OBJECT_TYPE_TRIGGER: + pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."), + "include", + filter_object_type_name(objtype)); + exit_nicely(1); + break; /* unreachable */ + + case FILTER_OBJECT_TYPE_EXTENSION: + simple_string_list_append(&extension_include_patterns, objname); + break; + case FILTER_OBJECT_TYPE_FOREIGN_DATA: + simple_string_list_append(&foreign_servers_include_patterns, objname); + break; + case FILTER_OBJECT_TYPE_SCHEMA: + simple_string_list_append(&schema_include_patterns, objname); + dopt->include_everything = false; + break; + case FILTER_OBJECT_TYPE_TABLE: + simple_string_list_append(&table_include_patterns, objname); + dopt->include_everything = false; + break; + case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN: + simple_string_list_append(&table_include_patterns_and_children, + objname); + dopt->include_everything = false; + break; + } + } + else if (comtype == FILTER_COMMAND_TYPE_EXCLUDE) + { + switch (objtype) + { + case FILTER_OBJECT_TYPE_NONE: + break; + case FILTER_OBJECT_TYPE_DATABASE: + case FILTER_OBJECT_TYPE_FUNCTION: + case FILTER_OBJECT_TYPE_INDEX: + case FILTER_OBJECT_TYPE_TRIGGER: + case FILTER_OBJECT_TYPE_EXTENSION: + case FILTER_OBJECT_TYPE_FOREIGN_DATA: + pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."), + "exclude", + filter_object_type_name(objtype)); + exit_nicely(1); + break; + + case FILTER_OBJECT_TYPE_TABLE_DATA: + simple_string_list_append(&tabledata_exclude_patterns, + objname); + break; + case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN: + simple_string_list_append(&tabledata_exclude_patterns_and_children, + objname); + break; + case FILTER_OBJECT_TYPE_SCHEMA: + simple_string_list_append(&schema_exclude_patterns, objname); + break; + case FILTER_OBJECT_TYPE_TABLE: + simple_string_list_append(&table_exclude_patterns, objname); + break; + case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN: + simple_string_list_append(&table_exclude_patterns_and_children, + objname); + break; + } + } + else + { + Assert(comtype == FILTER_COMMAND_TYPE_NONE); + Assert(objtype == FILTER_OBJECT_TYPE_NONE); + } + + if (objname) + free(objname); + } + + filter_free(&fstate); +} diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index e2a9733d348..1b974cf7e8e 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -26,6 +26,7 @@ #include "common/string.h" #include "dumputils.h" #include "fe_utils/string_utils.h" +#include "filter.h" #include "getopt_long.h" #include "pg_backup.h" @@ -81,6 +82,7 @@ static PGresult *executeQuery(PGconn *conn, const char *query); static void executeCommand(PGconn *conn, const char *query); static void expand_dbname_patterns(PGconn *conn, SimpleStringList *patterns, SimpleStringList *names); +static void read_dumpall_filters(const char *filename, SimpleStringList *patterns); static char pg_dump_bin[MAXPGPATH]; static const char *progname; @@ -177,6 +179,7 @@ main(int argc, char *argv[]) {"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1}, {"on-conflict-do-nothing", no_argument, &on_conflict_do_nothing, 1}, {"rows-per-insert", required_argument, NULL, 7}, + {"filter", required_argument, NULL, 8}, {NULL, 0, NULL, 0} }; @@ -360,6 +363,10 @@ main(int argc, char *argv[]) appendShellString(pgdumpopts, optarg); break; + case 8: + read_dumpall_filters(optarg, &database_exclude_patterns); + break; + default: /* getopt_long already emitted a complaint */ pg_log_error_hint("Try \"%s --help\" for more information.", progname); @@ -653,6 +660,7 @@ help(void) printf(_(" --disable-triggers disable triggers during data-only restore\n")); printf(_(" --exclude-database=PATTERN exclude databases whose name matches PATTERN\n")); printf(_(" --extra-float-digits=NUM override default setting for extra_float_digits\n")); + printf(_(" --filter=FILENAME exclude databases specified in FILENAME\n")); printf(_(" --if-exists use IF EXISTS when dropping objects\n")); printf(_(" --inserts dump data as INSERT commands, rather than COPY\n")); printf(_(" --load-via-partition-root load partitions via the root table\n")); @@ -1937,3 +1945,62 @@ hash_string_pointer(char *s) return hash_bytes(ss, strlen(s)); } + +/* + * read_dumpall_filters - retrieve database identifier patterns from file + * + * Parse the specified filter file for include and exclude patterns, and add + * them to the relevant lists. If the filename is "-" then filters will be + * read from STDIN rather than a file. + * + * At the moment, the only allowed filter is for database exclusion. + */ +static void +read_dumpall_filters(const char *filename, SimpleStringList *pattern) +{ + FilterStateData fstate; + char *objname; + FilterCommandType comtype; + FilterObjectType objtype; + + filter_init(&fstate, filename, exit); + + while (filter_read_item(&fstate, &objname, &comtype, &objtype)) + { + if (comtype == FILTER_COMMAND_TYPE_INCLUDE) + { + pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."), + "include", + filter_object_type_name(objtype)); + exit_nicely(1); + } + + switch (objtype) + { + case FILTER_OBJECT_TYPE_NONE: + break; + case FILTER_OBJECT_TYPE_FUNCTION: + case FILTER_OBJECT_TYPE_INDEX: + case FILTER_OBJECT_TYPE_TABLE_DATA: + case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN: + case FILTER_OBJECT_TYPE_TRIGGER: + case FILTER_OBJECT_TYPE_EXTENSION: + case FILTER_OBJECT_TYPE_FOREIGN_DATA: + case FILTER_OBJECT_TYPE_SCHEMA: + case FILTER_OBJECT_TYPE_TABLE: + case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN: + pg_log_filter_error(&fstate, _("unsupported filter object.")); + exit_nicely(1); + break; + + case FILTER_OBJECT_TYPE_DATABASE: + simple_string_list_append(pattern, objname); + break; + } + + if (objname) + free(objname); + } + + filter_free(&fstate); +} diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c index 049a1006347..1459e02263f 100644 --- a/src/bin/pg_dump/pg_restore.c +++ b/src/bin/pg_dump/pg_restore.c @@ -47,11 +47,13 @@ #include "dumputils.h" #include "fe_utils/option_utils.h" +#include "filter.h" #include "getopt_long.h" #include "parallel.h" #include "pg_backup_utils.h" static void usage(const char *progname); +static void read_restore_filters(const char *filename, RestoreOptions *dopt); int main(int argc, char **argv) @@ -123,6 +125,7 @@ main(int argc, char **argv) {"no-publications", no_argument, &no_publications, 1}, {"no-security-labels", no_argument, &no_security_labels, 1}, {"no-subscriptions", no_argument, &no_subscriptions, 1}, + {"filter", required_argument, NULL, 4}, {NULL, 0, NULL, 0} }; @@ -286,6 +289,10 @@ main(int argc, char **argv) set_dump_section(optarg, &(opts->dumpSections)); break; + case 4: + read_restore_filters(optarg, opts); + break; + default: /* getopt_long already emitted a complaint */ pg_log_error_hint("Try \"%s --help\" for more information.", progname); @@ -463,6 +470,8 @@ usage(const char *progname) printf(_(" -1, --single-transaction restore as a single transaction\n")); printf(_(" --disable-triggers disable triggers during data-only restore\n")); printf(_(" --enable-row-security enable row security\n")); + printf(_(" --filter=FILENAME restore or skip objects based on expressions\n" + " in FILENAME\n")); printf(_(" --if-exists use IF EXISTS when dropping objects\n")); printf(_(" --no-comments do not restore comments\n")); printf(_(" --no-data-for-failed-tables do not restore data of tables that could not be\n" @@ -494,3 +503,103 @@ usage(const char *progname) printf(_("Report bugs to <%s>.\n"), PACKAGE_BUGREPORT); printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL); } + +/* + * read_restore_filters - retrieve object identifier patterns from file + * + * Parse the specified filter file for include and exclude patterns, and add + * them to the relevant lists. If the filename is "-" then filters will be + * read from STDIN rather than a file. + */ +static void +read_restore_filters(const char *filename, RestoreOptions *opts) +{ + FilterStateData fstate; + char *objname; + FilterCommandType comtype; + FilterObjectType objtype; + + filter_init(&fstate, filename, exit_nicely); + + while (filter_read_item(&fstate, &objname, &comtype, &objtype)) + { + if (comtype == FILTER_COMMAND_TYPE_INCLUDE) + { + switch (objtype) + { + case FILTER_OBJECT_TYPE_NONE: + break; + case FILTER_OBJECT_TYPE_TABLE_DATA: + case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN: + case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN: + case FILTER_OBJECT_TYPE_DATABASE: + case FILTER_OBJECT_TYPE_EXTENSION: + case FILTER_OBJECT_TYPE_FOREIGN_DATA: + pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."), + "include", + filter_object_type_name(objtype)); + exit_nicely(1); + + case FILTER_OBJECT_TYPE_FUNCTION: + opts->selTypes = 1; + opts->selFunction = 1; + simple_string_list_append(&opts->functionNames, objname); + break; + case FILTER_OBJECT_TYPE_INDEX: + opts->selTypes = 1; + opts->selIndex = 1; + simple_string_list_append(&opts->indexNames, objname); + break; + case FILTER_OBJECT_TYPE_SCHEMA: + simple_string_list_append(&opts->schemaNames, objname); + break; + case FILTER_OBJECT_TYPE_TABLE: + opts->selTypes = 1; + opts->selTable = 1; + simple_string_list_append(&opts->tableNames, objname); + break; + case FILTER_OBJECT_TYPE_TRIGGER: + opts->selTypes = 1; + opts->selTrigger = 1; + simple_string_list_append(&opts->triggerNames, objname); + break; + } + } + else if (comtype == FILTER_COMMAND_TYPE_EXCLUDE) + { + switch (objtype) + { + case FILTER_OBJECT_TYPE_NONE: + break; + case FILTER_OBJECT_TYPE_TABLE_DATA: + case FILTER_OBJECT_TYPE_TABLE_DATA_AND_CHILDREN: + case FILTER_OBJECT_TYPE_DATABASE: + case FILTER_OBJECT_TYPE_EXTENSION: + case FILTER_OBJECT_TYPE_FOREIGN_DATA: + case FILTER_OBJECT_TYPE_FUNCTION: + case FILTER_OBJECT_TYPE_INDEX: + case FILTER_OBJECT_TYPE_TABLE: + case FILTER_OBJECT_TYPE_TABLE_AND_CHILDREN: + case FILTER_OBJECT_TYPE_TRIGGER: + pg_log_filter_error(&fstate, _("%s filter for \"%s\" is not allowed."), + "exclude", + filter_object_type_name(objtype)); + exit_nicely(1); + + case FILTER_OBJECT_TYPE_SCHEMA: + simple_string_list_append(&opts->schemaExcludeNames, objname); + break; + } + } + else + { + Assert(comtype == FILTER_COMMAND_TYPE_NONE); + Assert(objtype == FILTER_OBJECT_TYPE_NONE); + } + + if (objname) + free(objname); + } + + filter_free(&fstate); +} diff --git a/src/bin/pg_dump/t/005_pg_dump_filterfile.pl b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl new file mode 100644 index 00000000000..d16f034170f --- /dev/null +++ b/src/bin/pg_dump/t/005_pg_dump_filterfile.pl @@ -0,0 +1,799 @@ + +# Copyright (c) 2023, PostgreSQL Global Development Group + +use strict; +use warnings; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $tempdir = PostgreSQL::Test::Utils::tempdir; +my $inputfile; + +my $node = PostgreSQL::Test::Cluster->new('main'); +my $port = $node->port; +my $backupdir = $node->backup_dir; +my $plainfile = "$backupdir/plain.sql"; + +$node->init; +$node->start; + +# Generate test objects +$node->safe_psql('postgres', 'CREATE FOREIGN DATA WRAPPER dummy;'); +$node->safe_psql('postgres', + 'CREATE SERVER dummyserver FOREIGN DATA WRAPPER dummy;'); + +$node->safe_psql('postgres', "CREATE TABLE table_one(a varchar)"); +$node->safe_psql('postgres', "CREATE TABLE table_two(a varchar)"); +$node->safe_psql('postgres', "CREATE TABLE table_three(a varchar)"); +$node->safe_psql('postgres', "CREATE TABLE table_three_one(a varchar)"); +$node->safe_psql('postgres', "CREATE TABLE footab(a varchar)"); +$node->safe_psql('postgres', "CREATE TABLE bootab() inherits (footab)"); +$node->safe_psql( + 'postgres', "CREATE TABLE \"strange aaa +name\"(a varchar)"); +$node->safe_psql( + 'postgres', "CREATE TABLE \" +t +t +\"(a int)"); + +$node->safe_psql('postgres', + "INSERT INTO table_one VALUES('*** TABLE ONE ***')"); +$node->safe_psql('postgres', + "INSERT INTO table_two VALUES('*** TABLE TWO ***')"); +$node->safe_psql('postgres', + "INSERT INTO table_three VALUES('*** TABLE THREE ***')"); +$node->safe_psql('postgres', + "INSERT INTO table_three_one VALUES('*** TABLE THREE_ONE ***')"); +$node->safe_psql('postgres', "INSERT INTO bootab VALUES(10)"); + +$node->safe_psql('postgres', "CREATE DATABASE sourcedb"); +$node->safe_psql('postgres', "CREATE DATABASE targetdb"); + +$node->safe_psql('sourcedb', + 'CREATE FUNCTION foo1(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql' +); +$node->safe_psql('sourcedb', + 'CREATE FUNCTION foo2(a int) RETURNS int AS $$ select $1 $$ LANGUAGE sql' +); +$node->safe_psql('sourcedb', + 'CREATE FUNCTION foo3(a double precision, b int) RETURNS double precision AS $$ select $1 + $2 $$ LANGUAGE sql' +); +$node->safe_psql('sourcedb', + 'CREATE FUNCTION foo_trg() RETURNS trigger AS $$ BEGIN RETURN NEW; END $$ LANGUAGE plpgsql' +); +$node->safe_psql('sourcedb', 'CREATE SCHEMA s1'); +$node->safe_psql('sourcedb', 'CREATE SCHEMA s2'); +$node->safe_psql('sourcedb', 'CREATE TABLE s1.t1(a int)'); +$node->safe_psql('sourcedb', 'CREATE SEQUENCE s1.s1'); +$node->safe_psql('sourcedb', 'CREATE TABLE s2.t2(a int)'); +$node->safe_psql('sourcedb', 'CREATE TABLE t1(a int, b int)'); +$node->safe_psql('sourcedb', 'CREATE TABLE t2(a int, b int)'); +$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx1 ON t1(a)'); +$node->safe_psql('sourcedb', 'CREATE INDEX t1_idx2 ON t1(b)'); +$node->safe_psql('sourcedb', + 'CREATE TRIGGER trg1 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()'); +$node->safe_psql('sourcedb', + 'CREATE TRIGGER trg2 BEFORE INSERT ON t1 EXECUTE FUNCTION foo_trg()'); + +# +# Test interaction of correctly specified filter file +# +my ($cmd, $stdout, $stderr, $result); + +# Empty filterfile +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "\n # a comment and nothing more\n\n"; +close $inputfile; + +command_ok( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + "filter file without patterns"); + +my $dump = slurp_file($plainfile); + +ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "table one dumped"); +ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "table two dumped"); +ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "table three dumped"); +ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m, + "table three one dumped"); + +# Test various combinations of whitespace, comments and correct filters +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile " include table table_one #comment\n"; +print $inputfile "include table table_two\n"; +print $inputfile "# skip this line\n"; +print $inputfile "\n"; +print $inputfile "\t\n"; +print $inputfile " \t# another comment\n"; +print $inputfile "exclude table_data table_one\n"; +close $inputfile; + +command_ok( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + "dump tables with filter patterns as well as comments and whitespace"); + +$dump = slurp_file($plainfile); + +ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one"); +ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two"); +ok($dump !~ qr/^CREATE TABLE public\.table_three/m, "table three not dumped"); +ok($dump !~ qr/^CREATE TABLE public\.table_three_one/m, + "table three_one not dumped"); +ok( $dump !~ qr/^COPY public\.table_one/m, + "content of table one is not included"); +ok($dump =~ qr/^COPY public\.table_two/m, "content of table two is included"); + +# Test dumping tables specified by qualified names +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include table public.table_one\n"; +print $inputfile "include table \"public\".\"table_two\"\n"; +print $inputfile "include table \"public\". table_three\n"; +close $inputfile; + +command_ok( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + "filter file without patterns"); + +$dump = slurp_file($plainfile); + +ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "dumped table one"); +ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two"); +ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three"); + +# Test dumping all tables except one +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "exclude table table_one\n"; +close $inputfile; + +command_ok( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + "dump tables with exclusion of a single table"); + +$dump = slurp_file($plainfile); + +ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped"); +ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "dumped table two"); +ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three"); +ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m, + "dumped table three_one"); + +# Test dumping tables with a wildcard pattern +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include table table_thre*\n"; +close $inputfile; + +command_ok( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + "dump tables with wildcard in pattern"); + +$dump = slurp_file($plainfile); + +ok($dump !~ qr/^CREATE TABLE public\.table_one/m, "table one not dumped"); +ok($dump !~ qr/^CREATE TABLE public\.table_two/m, "table two not dumped"); +ok($dump =~ qr/^CREATE TABLE public\.table_three/m, "dumped table three"); +ok($dump =~ qr/^CREATE TABLE public\.table_three_one/m, + "dumped table three_one"); + +# Test dumping table with multiline quoted tablename +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include table \"strange aaa +name\""; +close $inputfile; + +command_ok( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + "dump tables with multiline names requiring quoting"); + +$dump = slurp_file($plainfile); + +ok($dump =~ qr/^CREATE TABLE public.\"strange aaa/m, + "dump table with new line in name"); + +# Test excluding multiline quoted tablename from dump +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "exclude table \"strange aaa\\nname\""; +close $inputfile; + +command_ok( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + "dump tables with filter"); + +$dump = slurp_file($plainfile); + +ok($dump !~ qr/^CREATE TABLE public.\"strange aaa/m, + "dump table with new line in name"); + +# Test excluding an entire schema +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "exclude schema public\n"; +close $inputfile; + +command_ok( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + "exclude the public schema"); + +$dump = slurp_file($plainfile); + +ok($dump !~ qr/^CREATE TABLE/m, "no table dumped"); + +# Test including and excluding an entire schema by multiple filterfiles +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include schema public\n"; +close $inputfile; + +open my $alt_inputfile, '>', "$tempdir/inputfile2.txt" + or die "unable to open filterfile for writing"; +print $alt_inputfile "exclude schema public\n"; +close $alt_inputfile; + +command_ok( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", + "--filter=$tempdir/inputfile2.txt", 'postgres' + ], + "exclude the public schema with multiple filters"); + +$dump = slurp_file($plainfile); + +ok($dump !~ qr/^CREATE TABLE/m, "no table dumped"); + +# Test dumping a table with a single leading newline on a row +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include table \" +t +t +\""; +close $inputfile; + +command_ok( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + "dump tables with filter"); + +$dump = slurp_file($plainfile); + +ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms, + "dump table with multiline strange name"); + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include table \"\\nt\\nt\\n\""; +close $inputfile; + +command_ok( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + "dump tables with filter"); + +$dump = slurp_file($plainfile); + +ok($dump =~ qr/^CREATE TABLE public.\"\nt\nt\n\" \($/ms, + "dump table with multiline strange name"); + +######################################### +# Test foreign_data + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include foreign_data doesnt_exists\n"; +close $inputfile; + +command_fails_like( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + qr/pg_dump: error: no matching foreign servers were found for pattern/, + "dump nonexisting foreign server"); + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile, "include foreign_data dummyserver\n"; +close $inputfile; + +command_ok( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + "dump foreign_data with filter"); + +$dump = slurp_file($plainfile); + +ok($dump =~ qr/^CREATE SERVER dummyserver/m, "dump foreign server"); + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "exclude foreign_data dummy*\n"; +close $inputfile; + +command_fails_like( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + qr/exclude filter for "foreign data" is not allowed/, + "erroneously exclude foreign server"); + +######################################### +# Test broken input format + +# Test invalid filter command +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "k"; +close $inputfile; + +command_fails_like( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + qr/invalid filter command/, + "invalid syntax: incorrect filter command"); + +# Test invalid object type +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include xxx"; +close $inputfile; + +command_fails_like( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + qr/unsupported filter object type: "xxx"/, + "invalid syntax: invalid object type specified, should be table, schema, foreign_data or data" +); + +# Test missing object identifier pattern +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include table"; +close $inputfile; + +command_fails_like( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + qr/missing object name/, + "invalid syntax: missing object identifier pattern"); + +# Test adding extra content after the object identifier pattern +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include table table one"; +close $inputfile; + +command_fails_like( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + qr/no matching tables were found/, + "invalid syntax: extra content after object identifier pattern"); + +######################################### +# Combined with --strict-names + +# First ensure that a matching filter works +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include table table_one\n"; +close $inputfile; + +command_ok( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", + '--strict-names', 'postgres' + ], + "strict names with matching pattern"); + +$dump = slurp_file($plainfile); + +ok($dump =~ qr/^CREATE TABLE public\.table_one/m, "no table dumped"); + +# Now append a pattern to the filter file which doesn't resolve +open $inputfile, '>>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include table table_nonexisting_name"; +close $inputfile; + +command_fails_like( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", + '--strict-names', 'postgres' + ], + qr/no matching tables were found/, + "inclusion of non-existing objects with --strict names"); + +######################################### +# pg_dumpall tests + +########################### +# Test dumping all tables except one +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "exclude database postgres\n"; +close $inputfile; + +command_ok( + [ + 'pg_dumpall', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt" + ], + "dump tables with exclusion of a database"); + +$dump = slurp_file($plainfile); + +ok($dump !~ qr/^\\connect postgres/m, "database postgres is not dumped"); +ok($dump =~ qr/^\\connect template1/m, "database template1 is dumped"); + +# Make sure this option dont break the existing limitation of using +# --globals-only with exclusions +command_fails_like( + [ + 'pg_dumpall', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", + '--globals-only' + ], + qr/\Qpg_dumpall: error: option --exclude-database cannot be used together with -g\/--globals-only\E/, + 'pg_dumpall: option --exclude-database cannot be used together with -g/--globals-only' +); + +# Test invalid filter command +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "k"; +close $inputfile; + +command_fails_like( + [ + 'pg_dumpall', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt" + ], + qr/invalid filter command/, + "invalid syntax: incorrect filter command"); + +# Test invalid object type +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "exclude xxx"; +close $inputfile; + +command_fails_like( + [ + 'pg_dumpall', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt" + ], + qr/unsupported filter object type: "xxx"/, + "invalid syntax: exclusion of non-existing object type"); + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "exclude table foo"; +close $inputfile; + +command_fails_like( + [ + 'pg_dumpall', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt" + ], + qr/pg_dumpall: error: invalid format in filter/, + "invalid syntax: exclusion of unsupported object type"); + +######################################### +# pg_restore tests + +command_ok( + [ + 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump", + "-Fc", 'postgres' + ], + "dump all tables"); + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include table table_two"; +close $inputfile; + +command_ok( + [ + 'pg_restore', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", + "-Fc", "$tempdir/filter_test.dump" + ], + "restore tables with filter"); + +$dump = slurp_file($plainfile); + +ok($dump =~ qr/^CREATE TABLE public\.table_two/m, "wanted table restored"); +ok($dump !~ qr/^CREATE TABLE public\.table_one/m, + "unwanted table is not restored"); + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include table_data xxx"; +close $inputfile; + +command_fails_like( + [ + 'pg_restore', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt" + ], + qr/include filter for "table data" is not allowed/, + "invalid syntax: inclusion of unallowed object"); + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include extension xxx"; +close $inputfile; + +command_fails_like( + [ + 'pg_restore', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt" + ], + qr/include filter for "extension" is not allowed/, + "invalid syntax: inclusion of unallowed object"); + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "exclude extension xxx"; +close $inputfile; + +command_fails_like( + [ + 'pg_restore', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt" + ], + qr/exclude filter for "extension" is not allowed/, + "invalid syntax: exclusion of unallowed object"); + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "exclude table_data xxx"; +close $inputfile; + +command_fails_like( + [ + 'pg_restore', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt" + ], + qr/exclude filter for "table data" is not allowed/, + "invalid syntax: exclusion of unallowed object"); + +######################################### +# test restore of other objects + +command_ok( + [ + 'pg_dump', '-p', $port, '-f', "$tempdir/filter_test.dump", + "-Fc", 'sourcedb' + ], + "dump all objects from sourcedb"); + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include function foo1(integer)"; +close $inputfile; + +command_ok( + [ + 'pg_restore', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", + "-Fc", "$tempdir/filter_test.dump" + ], + "restore function with filter"); + +$dump = slurp_file($plainfile); + +ok($dump =~ qr/^CREATE FUNCTION public\.foo1/m, "wanted function restored"); +ok( $dump !~ qr/^CREATE TABLE public\.foo2/m, + "unwanted function is not restored"); + +# this should be white space tolerant (against the -P argument) + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include function foo3 ( double precision , integer) "; +close $inputfile; + +command_ok( + [ + 'pg_restore', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", + "-Fc", "$tempdir/filter_test.dump" + ], + "restore function with filter"); + +$dump = slurp_file($plainfile); + +ok($dump =~ qr/^CREATE FUNCTION public\.foo3/m, "wanted function restored"); + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include index t1_idx1\n"; + +# attention! this hit pg_restore bug - correct name of trigger is "trg1" +# not "t1 trg1". Should be fixed when pg_restore will be fixed +print $inputfile "include trigger t1 trg1\n"; +close $inputfile; + +command_ok( + [ + 'pg_restore', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", + "-Fc", "$tempdir/filter_test.dump" + ], + "restore function with filter"); + +$dump = slurp_file($plainfile); + +ok($dump =~ qr/^CREATE INDEX t1_idx1/m, "wanted index restored"); +ok($dump !~ qr/^CREATE INDEX t2_idx2/m, "unwanted index are not restored"); +ok($dump =~ qr/^CREATE TRIGGER trg1/m, "wanted trigger restored"); +ok($dump !~ qr/^CREATE TRIGGER trg2/m, "unwanted trigger is not restored"); + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include schema s1\n"; +close $inputfile; + +command_ok( + [ + 'pg_restore', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", + "-Fc", "$tempdir/filter_test.dump" + ], + "restore function with filter"); + +$dump = slurp_file($plainfile); + +ok($dump =~ qr/^CREATE TABLE s1\.t1/m, "wanted table from schema restored"); +ok( $dump =~ qr/^CREATE SEQUENCE s1\.s1/m, + "wanted sequence from schema restored"); +ok($dump !~ qr/^CREATE TABLE s2\t2/m, "unwanted table is not restored"); + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "exclude schema s1\n"; +close $inputfile; + +command_ok( + [ + 'pg_restore', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", + "-Fc", "$tempdir/filter_test.dump" + ], + "restore function with filter"); + +$dump = slurp_file($plainfile); + +ok($dump !~ qr/^CREATE TABLE s1\.t1/m, + "unwanted table from schema is not restored"); +ok($dump !~ qr/^CREATE SEQUENCE s1\.s1/m, + "unwanted sequence from schema is not restored"); +ok($dump =~ qr/^CREATE TABLE s2\.t2/m, "wanted table restored"); +ok($dump =~ qr/^CREATE TABLE public\.t1/m, "wanted table restored"); + +######################################### +# test of supported syntax + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; + +print $inputfile "include table_and_children footab\n"; +close $inputfile; + +command_ok( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + "filter file without patterns"); + +$dump = slurp_file($plainfile); + +ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table"); + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; + +print $inputfile "exclude table_and_children footab\n"; +close $inputfile; + +command_ok( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + "filter file without patterns"); + +$dump = slurp_file($plainfile); + +ok($dump !~ qr/^CREATE TABLE public\.bootab/m, + "exclude dumped children table"); + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; + +print $inputfile "exclude table_data_and_children footab\n"; +close $inputfile; + +command_ok( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + "filter file without patterns"); + +$dump = slurp_file($plainfile); + +ok($dump =~ qr/^CREATE TABLE public\.bootab/m, "dumped children table"); +ok($dump !~ qr/^COPY public\.bootab/m, "exclude dumped children table"); + +######################################### +# Test extension + +open $inputfile, '>', "$tempdir/inputfile.txt" + or die "unable to open filterfile for writing"; +print $inputfile "include extension doesnt_exists\n"; +close $inputfile; + +command_fails_like( + [ + 'pg_dump', '-p', $port, '-f', $plainfile, + "--filter=$tempdir/inputfile.txt", 'postgres' + ], + qr/pg_dump: error: no matching extensions were found/, + "dump nonexisting extension"); + + +done_testing(); diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index d26d1a84e81..46df01cc8d2 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -455,6 +455,7 @@ sub mkvcbuild $pgdumpall->AddIncludeDir('src/backend'); $pgdumpall->AddFile('src/bin/pg_dump/pg_dumpall.c'); $pgdumpall->AddFile('src/bin/pg_dump/dumputils.c'); + $pgdumpall->AddFile('src/bin/pg_dump/filter.c'); $pgdumpall->AddLibrary('ws2_32.lib'); my $pgrestore = AddSimpleFrontend('pg_dump', 1); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 86a9886d4f7..d659adbfd6c 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -745,6 +745,9 @@ FileFdwPlanState FileNameMap FileSet FileTag +FilterCommandType +FilterObjectType +FilterStateData FinalPathExtraData FindColsContext FindSplitData