database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-07-30 16:24:05 +03:00
Code Activity

pre-MDEV-30157 & pre-MDEV-28669: fixes before the main corrections

Browse Source 
This commit adds even more correct handling of parameters
with paths when they contain leading or trailing spaces and/or
slashes. Also it fixes problems that occur when the user specified
explicit paths to additional directories, but these paths match
the specified path of the data directory - in this case, additional
subdirectories should be treated (in relation to the data directory)
in the same way as if these paths were not specified or as if they
are implicitly specified as "." or "./". But prior to this fix,
existing code treated any values as if they were completely
separate directories, whether or not they actually point to the
same location to which datadir points to - and this sometimes
resulted in incorrect file transfers.

This fix does not contain separate tests, as tests will be
part of the main commit(s). This fix has been made as a separate
commit to facilitate review for major substantive fixes related
to MDEV-30157 and MDEV-28669.
This commit is contained in:
Julius Goryavsky
2022-12-13 10:32:21 +01:00
parent b928c849d2
commit e4a4aad7cf
3 changed files with 118 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
scripts/wsrep_sst_common.sh
View File

@ -47,18 +47,51 @@ trim_string()
trim_dir() trim_dir()
{ {
local t=$(trim_string "$1")
if [ "$t" != '/' ]; then
if [ "${t%/}" != "$t" ]; then
t=$(trim_string "${t%/}")
fi
else
t='.'
fi
if [ -n "$BASH_VERSION" ]; then if [ -n "$BASH_VERSION" ]; then
printf '%s' "$t" local pattern="![:space:]${2:-}"
local x="${1#*[$pattern]}"
local z=${#1}
x=${#x}
if [ $x -ne $z ]; then
local y="${1%[$pattern/]*}"
y=${#y}
x=$(( z-x-1 ))
y=$(( y-x+1 ))
x="${1:$x:$y}"
[ -z "$x" ] && x='.'
printf '%s' "$x"
else else
echo "$t" printf ''
fi
else
local pattern="[:space:]${2:-}"
local x=$(echo "$1" | sed -E "s/^[$pattern]+|[$pattern/]+\$//g")
if [ -n "$x" ]; then
echo "$x"
elif "${1#*/}" != "$1"; then
echo '.'
else
echo ''
fi
fi
}
trim_right()
{
if [ -n "$BASH_VERSION" ]; then
local pattern="[![:space:]${2:-}]"
local z=${#1}
local y="${1%$pattern*}"
y=${#y}
if [ $y -ne $z ]; then
y=$(( y+1 ))
printf '%s' "${1:0:$y}"
else
printf ''
fi
else
local pattern="[[:space:]${2:-}]"
echo "$1" | sed -E "s/$pattern+\$//g"
fi fi
} }
@ -111,7 +144,7 @@ INNOEXTRA=""
while [ $# -gt 0 ]; do while [ $# -gt 0 ]; do
case "$1" in case "$1" in
'--address') '--address')
WSREP_SST_OPT_ADDR="$2" WSREP_SST_OPT_ADDR=$(trim_string "$2")
# #
# Break address string into host:port/path parts # Break address string into host:port/path parts
# #
@ -119,20 +152,22 @@ case "$1" in
\[*) \[*)
# IPv6 # IPv6
# Remove the starting and ending square brackets, if present: # Remove the starting and ending square brackets, if present:
addr_no_bracket="${WSREP_SST_OPT_ADDR#\[}" addr="${WSREP_SST_OPT_ADDR#\[}"
addr=$(trim_right "${addr%%\]*}")
# Some utilities and subsequent code require an address # Some utilities and subsequent code require an address
# without square brackets: # without square brackets:
readonly WSREP_SST_OPT_HOST_UNESCAPED="${addr_no_bracket%%\]*}" readonly WSREP_SST_OPT_HOST_UNESCAPED="$addr"
# Square brackets are needed in most cases: # Square brackets are needed in most cases:
readonly WSREP_SST_OPT_HOST="[$WSREP_SST_OPT_HOST_UNESCAPED]" readonly WSREP_SST_OPT_HOST="[$addr]"
# Mark this address as IPv6: # Mark this address as IPv6:
readonly WSREP_SST_OPT_HOST_IPv6=1 readonly WSREP_SST_OPT_HOST_IPv6=1
# Let's remove the leading part that contains the host address: # Let's remove the leading part that contains the host address:
remain="${WSREP_SST_OPT_ADDR#*\]}" remain="${WSREP_SST_OPT_ADDR#*\]}"
;; ;;
*) *)
readonly WSREP_SST_OPT_HOST="${WSREP_SST_OPT_ADDR%%[:/]*}" addr=$(trim_right "${WSREP_SST_OPT_ADDR%%[:/]*}")
readonly WSREP_SST_OPT_HOST_UNESCAPED="$WSREP_SST_OPT_HOST" readonly WSREP_SST_OPT_HOST="$addr"
readonly WSREP_SST_OPT_HOST_UNESCAPED="$addr"
readonly WSREP_SST_OPT_HOST_IPv6=0 readonly WSREP_SST_OPT_HOST_IPv6=0
# Let's remove the leading part that contains the host address: # Let's remove the leading part that contains the host address:
remain="${WSREP_SST_OPT_ADDR#*[:/]}" remain="${WSREP_SST_OPT_ADDR#*[:/]}"
@ -154,17 +189,18 @@ case "$1" in
else else
readonly WSREP_SST_OPT_PATH="" readonly WSREP_SST_OPT_PATH=""
fi fi
WSREP_SST_OPT_ADDR_PORT=$(trim_right "$WSREP_SST_OPT_ADDR_PORT")
# Remove the module name part from the string, which ends with "/": # Remove the module name part from the string, which ends with "/":
remain="${WSREP_SST_OPT_PATH#*/}" remain="${WSREP_SST_OPT_PATH#*/}"
# This operation removes the tail after the very first occurrence # This operation removes the tail after the very first occurrence
# of the "/" character, inclusively: # of the "/" character, inclusively:
readonly WSREP_SST_OPT_MODULE="${WSREP_SST_OPT_PATH%%/*}" readonly WSREP_SST_OPT_MODULE=$(trim_right "${WSREP_SST_OPT_PATH%%/*}")
# If there is one more "/" in the string, then everything before # If there is one more "/" in the string, then everything before
# it will be the LSN, otherwise the LSN is empty: # it will be the LSN, otherwise the LSN is empty:
if [ "$remain" != "$WSREP_SST_OPT_PATH" ]; then if [ "$remain" != "$WSREP_SST_OPT_PATH" ]; then
# Extract the part that matches the LSN by removing all # Extract the part that matches the LSN by removing all
# characters starting from the very first "/": # characters starting from the very first "/":
readonly WSREP_SST_OPT_LSN="${remain%%/*}" readonly WSREP_SST_OPT_LSN=$(trim_right "${remain%%/*}")
# Exctract everything after the first occurrence of # Exctract everything after the first occurrence of
# the "/" character in the string: # the "/" character in the string:
source="$remain" source="$remain"
@ -176,7 +212,7 @@ case "$1" in
# Let's extract the version number by removing the tail # Let's extract the version number by removing the tail
# after the very first occurence of the "/" character # after the very first occurence of the "/" character
# (inclusively): # (inclusively):
readonly WSREP_SST_OPT_SST_VER="${remain%%/*}" readonly WSREP_SST_OPT_SST_VER=$(trim_right "${remain%%/*}")
else else
readonly WSREP_SST_OPT_SST_VER="" readonly WSREP_SST_OPT_SST_VER=""
fi fi
@ -218,41 +254,46 @@ case "$1" in
shift shift
;; ;;
'--defaults-file') '--defaults-file')
readonly WSREP_SST_OPT_DEFAULT="$1=$2" file=$(trim_string "$2")
readonly WSREP_SST_OPT_DEFAULTS="$1='$2'" readonly WSREP_SST_OPT_DEFAULT="$1=$file"
readonly WSREP_SST_OPT_DEFAULTS="$1='$file'"
shift shift
;; ;;
'--defaults-extra-file') '--defaults-extra-file')
readonly WSREP_SST_OPT_EXTRA_DEFAULT="$1=$2" file=$(trim_string "$2")
readonly WSREP_SST_OPT_EXTRA_DEFAULTS="$1='$2'" readonly WSREP_SST_OPT_EXTRA_DEFAULT="$1=$file"
readonly WSREP_SST_OPT_EXTRA_DEFAULTS="$1='$file'"
shift shift
;; ;;
'--defaults-group-suffix') '--defaults-group-suffix')
readonly WSREP_SST_OPT_SUFFIX_DEFAULT="$1=$2" suffix=$(trim_string "$2")
readonly WSREP_SST_OPT_SUFFIX_VALUE="$2" readonly WSREP_SST_OPT_SUFFIX_DEFAULT="$1=$suffix"
readonly WSREP_SST_OPT_SUFFIX_VALUE="$suffix"
shift shift
;; ;;
'--host') '--host')
case "$2" in addr=$(trim_string "$2")
case "$addr" in
\[*) \[*)
# IPv6 # IPv6
# Remove the starting and ending square brackets, if present: # Remove the starting and ending square brackets, if present:
addr_no_bracket="${2#\[}" addr="${addr#\[}"
addr=$(trim_right "${addr%%\]*}")
# Some utilities and subsequent code require an address # Some utilities and subsequent code require an address
# without square brackets: # without square brackets:
readonly WSREP_SST_OPT_HOST_UNESCAPED="${addr_no_bracket%%\]*}" readonly WSREP_SST_OPT_HOST_UNESCAPED="$addr"
# Square brackets are needed in most cases: # Square brackets are needed in most cases:
readonly WSREP_SST_OPT_HOST="[${WSREP_SST_OPT_HOST_UNESCAPED}]" readonly WSREP_SST_OPT_HOST="[$addr]"
# Mark this address as IPv6: # Mark this address as IPv6:
readonly WSREP_SST_OPT_HOST_IPv6=1 readonly WSREP_SST_OPT_HOST_IPv6=1
;; ;;
*) *)
readonly WSREP_SST_OPT_HOST="$2" readonly WSREP_SST_OPT_HOST="$addr"
readonly WSREP_SST_OPT_HOST_UNESCAPED="$2" readonly WSREP_SST_OPT_HOST_UNESCAPED="$addr"
readonly WSREP_SST_OPT_HOST_IPv6=0 readonly WSREP_SST_OPT_HOST_IPv6=0
;; ;;
esac esac
WSREP_SST_OPT_ADDR="$WSREP_SST_OPT_HOST" WSREP_SST_OPT_ADDR="$addr"
shift shift
;; ;;
'--local-port') '--local-port')
@ -272,11 +313,11 @@ case "$1" in
shift shift
;; ;;
'--role') '--role')
readonly WSREP_SST_OPT_ROLE="$2" readonly WSREP_SST_OPT_ROLE=$(trim_string "$2")
shift shift
;; ;;
'--socket') '--socket')
readonly WSREP_SST_OPT_SOCKET="$2" readonly WSREP_SST_OPT_SOCKET=$(trim_string "$2")
shift shift
;; ;;
'--user') '--user')
@ -284,23 +325,23 @@ case "$1" in
shift shift
;; ;;
'--gtid') '--gtid')
readonly WSREP_SST_OPT_GTID="$2" readonly WSREP_SST_OPT_GTID=$(trim_string "$2")
shift shift
;; ;;
'--binlog'|'--log-bin') '--binlog'|'--log-bin')
readonly WSREP_SST_OPT_BINLOG="$2" readonly WSREP_SST_OPT_BINLOG=$(trim_string "$2")
shift shift
;; ;;
'--binlog-index'|'--log-bin-index') '--binlog-index'|'--log-bin-index')
WSREP_SST_OPT_BINLOG_INDEX="$2" WSREP_SST_OPT_BINLOG_INDEX=$(trim_string "$2")
shift shift
;; ;;
'--log-basename') '--log-basename')
readonly WSREP_SST_OPT_LOG_BASENAME="$2" readonly WSREP_SST_OPT_LOG_BASENAME=$(trim_string "$2")
shift shift
;; ;;
'--gtid-domain-id') '--gtid-domain-id')
readonly WSREP_SST_OPT_GTID_DOMAIN_ID="$2" readonly WSREP_SST_OPT_GTID_DOMAIN_ID=$(trim_string "$2")
shift shift
;; ;;
'--mysqld-args') '--mysqld-args')

28
scripts/wsrep_sst_mariabackup.sh
View File

@ -439,9 +439,10 @@ get_footprint()
-regex '.*undo[0-9]+$\|.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' \ -regex '.*undo[0-9]+$\|.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' \
-type f -print0 | du --files0-from=- --block-size=1 -c -s | \ -type f -print0 | du --files0-from=- --block-size=1 -c -s | \
awk 'END { print $1 }') awk 'END { print $1 }')
local payload_undo=0 local payload_undo=0
if [ -n "$ib_undo_dir" -a -d "$ib_undo_dir" ]; then if [ -n "$ib_undo_dir" -a "$ib_undo_dir" != '.' -a \
"$ib_undo_dir" != "$DATA_DIR" -a -d "$ib_undo_dir" ]
then
cd "$ib_undo_dir" cd "$ib_undo_dir"
payload_undo=$(find . -regex '.*undo[0-9]+$' -type f -print0 | \ payload_undo=$(find . -regex '.*undo[0-9]+$' -type f -print0 | \
du --files0-from=- --block-size=1 -c -s | awk 'END { print $1 }') du --files0-from=- --block-size=1 -c -s | awk 'END { print $1 }')
@ -1220,13 +1221,16 @@ else # joiner
INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR")
fi fi
if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' -a \
"$INNODB_DATA_HOME_DIR" != "$DATA_DIR" ]
then
# handle both relative and absolute paths: # handle both relative and absolute paths:
cd "$DATA" cd "$DATA"
[ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR"
cd "$INNODB_DATA_HOME_DIR" cd "$INNODB_DATA_HOME_DIR"
ib_home_dir="$(pwd)" ib_home_dir="$(pwd)"
cd "$OLD_PWD" cd "$OLD_PWD"
[ "$ib_home_dir" = "$DATA_DIR" ] && ib_home_dir=""
fi fi
# if no command line argument and INNODB_LOG_GROUP_HOME is not set, # if no command line argument and INNODB_LOG_GROUP_HOME is not set,
@ -1236,13 +1240,16 @@ else # joiner
INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME")
fi fi
if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' -a \
"$INNODB_LOG_GROUP_HOME" != "$DATA_DIR" ]
then
# handle both relative and absolute paths: # handle both relative and absolute paths:
cd "$DATA" cd "$DATA"
[ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME"
cd "$INNODB_LOG_GROUP_HOME" cd "$INNODB_LOG_GROUP_HOME"
ib_log_dir="$(pwd)" ib_log_dir="$(pwd)"
cd "$OLD_PWD" cd "$OLD_PWD"
[ "$ib_log_dir" = "$DATA_DIR" ] && ib_log_dir=""
fi fi
# if no command line argument and INNODB_UNDO_DIR is not set, # if no command line argument and INNODB_UNDO_DIR is not set,
@ -1252,13 +1259,16 @@ else # joiner
INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR")
fi fi
if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' -a \
"$INNODB_UNDO_DIR" != "$DATA_DIR" ]
then
# handle both relative and absolute paths: # handle both relative and absolute paths:
cd "$DATA" cd "$DATA"
[ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR"
cd "$INNODB_UNDO_DIR" cd "$INNODB_UNDO_DIR"
ib_undo_dir="$(pwd)" ib_undo_dir="$(pwd)"
cd "$OLD_PWD" cd "$OLD_PWD"
[ "$ib_undo_dir" = "$DATA_DIR" ] && ib_undo_dir=""
fi fi
if [ -n "$backup_threads" ]; then if [ -n "$backup_threads" ]; then
@ -1500,11 +1510,15 @@ else # joiner
binlogs=$(ls -d -1 "$binlog_base".[0-9]* 2>/dev/null || :) binlogs=$(ls -d -1 "$binlog_base".[0-9]* 2>/dev/null || :)
fi fi
cd "$DATA_DIR" cd "$DATA_DIR"
if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \
"$binlog_dir" != "$DATA_DIR" ]
then
[ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir"
fi fi
index_dir=$(dirname "$binlog_index"); index_dir=$(dirname "$binlog_index");
if [ -n "$index_dir" -a "$index_dir" != '.' ]; then if [ -n "$index_dir" -a "$index_dir" != '.' -a \
"$index_dir" != "$DATA_DIR" ]
then
[ ! -d "$index_dir" ] && mkdir -p "$index_dir" [ ! -d "$index_dir" ] && mkdir -p "$index_dir"
fi fi
if [ -n "$binlogs" ]; then if [ -n "$binlogs" ]; then

26
scripts/wsrep_sst_rsync.sh
View File

@ -185,7 +185,9 @@ if [ -z "$INNODB_LOG_GROUP_HOME" ]; then
INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME")
fi fi
if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' -a \
"$INNODB_LOG_GROUP_HOME" != "$DATA_DIR" ]
then
# handle both relative and absolute paths: # handle both relative and absolute paths:
cd "$DATA" cd "$DATA"
[ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME"
@ -201,7 +203,9 @@ if [ -z "$INNODB_DATA_HOME_DIR" ]; then
INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR")
fi fi
if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' -a \
"$INNODB_DATA_HOME_DIR" != "$DATA_DIR" ]
then
# handle both relative and absolute paths: # handle both relative and absolute paths:
cd "$DATA" cd "$DATA"
[ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR"
@ -217,7 +221,9 @@ if [ -z "$INNODB_UNDO_DIR" ]; then
INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR")
fi fi
if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' -a \
"$INNODB_UNDO_DIR" != "$DATA_DIR" ]
then
# handle both relative and absolute paths: # handle both relative and absolute paths:
cd "$DATA" cd "$DATA"
[ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR"
@ -504,7 +510,9 @@ EOF
if [ "$first" = '-' -o "$first" = '@' ]; then if [ "$first" = '-' -o "$first" = '@' ]; then
bin_base="./$bin_base" bin_base="./$bin_base"
fi fi
if [ -n "$bin_dir" -a "$bin_dir" != '.' ]; then if [ -n "$bin_dir" -a "$bin_dir" != '.' -a \
"$bin_dir" != "$DATA_DIR" ]
then
tar $tar_options "$BINLOG_TAR_FILE" \ tar $tar_options "$BINLOG_TAR_FILE" \
-C "$bin_dir" "$bin_base" >&2 -C "$bin_dir" "$bin_base" >&2
else else
@ -872,7 +880,7 @@ EOF
binlog_cd=0 binlog_cd=0
# Change the directory to binlog base (if possible): # Change the directory to binlog base (if possible):
if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \ if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \
-d "$binlog_dir" ] "$binlog_dir" != "$DATA_DIR" -a -d "$binlog_dir" ]
then then
binlog_cd=1 binlog_cd=1
cd "$binlog_dir" cd "$binlog_dir"
@ -891,11 +899,15 @@ EOF
tmpfile=$(TMPDIR="$tmpdir"; mktemp) tmpfile=$(TMPDIR="$tmpdir"; mktemp)
fi fi
index_dir=$(dirname "$binlog_index"); index_dir=$(dirname "$binlog_index");
if [ -n "$index_dir" -a "$index_dir" != '.' ]; then if [ -n "$index_dir" -a "$index_dir" != '.' -a \
"$index_dir" != "$DATA_DIR" ]
then
[ ! -d "$index_dir" ] && mkdir -p "$index_dir" [ ! -d "$index_dir" ] && mkdir -p "$index_dir"
fi fi
binlog_cd=0 binlog_cd=0
if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \
"$binlog_dir" != "$DATA_DIR" ]
then
[ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir"
binlog_cd=1 binlog_cd=1
cd "$binlog_dir" cd "$binlog_dir"