diff --git a/.gitignore b/.gitignore
index 469aa0d894b..b5cc5abdb0b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -49,6 +49,8 @@ extra/comp_err
extra/innochecksum
extra/jemalloc/build/
extra/jemalloc/tmp/
+extra/mariabackup/mariabackup
+extra/mariabackup/mbstream
extra/my_print_defaults
extra/mysql_waitpid
extra/mysqld_safe_helper
@@ -124,10 +126,13 @@ scripts/mytop
scripts/wsrep_sst_common
scripts/wsrep_sst_mysqldump
scripts/wsrep_sst_rsync
+scripts/wsrep_sst_mariabackup
scripts/wsrep_sst_xtrabackup
scripts/wsrep_sst_xtrabackup-v2
scripts/maria_add_gis_sp.sql
scripts/maria_add_gis_sp_bootstrap.sql
+scripts/galera_new_cluster
+scripts/galera_recovery
sql-bench/bench-count-distinct
sql-bench/bench-init.pl
sql-bench/compare-results
@@ -219,6 +224,7 @@ support-files/config.medium.ini
support-files/config.small.ini
support-files/mariadb.pc
support-files/mariadb@.service
+support-files/mariadb.service
support-files/my-huge.cnf
support-files/my-innodb-heavy-4G.cnf
support-files/my-large.cnf
@@ -232,6 +238,7 @@ support-files/mysqld_multi.server
support-files/wsrep.cnf
support-files/wsrep_notify
support-files/policy/selinux/mysqld-safe.pp
+support-files/mariadb.pp
tags
tests/async_queries
tests/bug25714
@@ -248,6 +255,10 @@ storage/mroonga/vendor/groonga/src/groonga-benchmark
storage/mroonga/vendor/groonga/src/suggest/groonga-suggest-create-dataset
storage/mroonga/mysql-test/mroonga/storage/r/information_schema_plugins.result
storage/mroonga/mysql-test/mroonga/storage/r/variable_version.result
+xxx/*
+yyy/*
+zzz/*
+
# C and C++
# Compiled Object files
diff --git a/.travis.compiler.sh b/.travis.compiler.sh
index 35e79e177ef..13e35fffe87 100755
--- a/.travis.compiler.sh
+++ b/.travis.compiler.sh
@@ -8,8 +8,8 @@ if [[ "${TRAVIS_OS_NAME}" == 'linux' ]]; then
CMAKE_OPT="${CMAKE_OPT} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
fi
case ${GCC_VERSION} in
- 5) CXX=clang++-3.9 ;;
- 6) CXX=clang++-4.0 ;;
+ 5) CXX=clang++-4.0 ;;
+ 6) CXX=clang++-5.0 ;;
esac
export CXX CC=${CXX/++/}
elif [[ "${CXX}" == 'g++' ]]; then
@@ -39,8 +39,4 @@ else
fi
fi
-# main.mysqlhotcopy_myisam consitently failed in travis containers
-# https://travis-ci.org/grooverdan/mariadb-server/builds/217661580
-echo 'main.mysqlhotcopy_myisam : unstable in containers' >> ${TRAVIS_BUILD_DIR}/mysql-test/unstable-tests
-echo 'archive.mysqlhotcopy_archive : unstable in containers' >> ${TRAVIS_BUILD_DIR}/mysql-test/unstable-tests
set +v +x
diff --git a/.travis.yml b/.travis.yml
index f33de076289..f3253b5e1a9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -40,15 +40,66 @@ matrix:
include:
- os: linux
compiler: gcc
+ addons:
+ apt:
+ packages: # make sure these match debian/control contents
+ - bison
+ - chrpath
+ - cmake
+ - debhelper
+ - dh-apparmor
+ - dpatch
+ - gdb
+ - libaio-dev
+ - libboost-dev
+ - libcurl3-dev
+ - libdbd-mysql
+ - libjudy-dev
+ - libncurses5-dev
+ - libpam0g-dev
+ - libpcre3-dev
+ - libreadline-gplv2-dev
+ - libstemmer-dev
+ - libssl-dev
+ - libnuma-dev
+ - libxml2-dev
+ - lsb-release
+ - perl
+ - po-debconf
+ - psmisc
+ - zlib1g-dev
+ - libcrack2-dev
+ - cracklib-runtime
+ - libjemalloc-dev
+ - libsnappy-dev
+ - liblzma-dev
+ - libzmq-dev
+ - libdistro-info-perl
+ - uuid-dev
+ - devscripts
+ - fakeroot
script:
- ${CC} --version ; ${CXX} --version
- # Just for disabling hotcopy tests for now
- source .travis.compiler.sh
# https://github.com/travis-ci/travis-ci/issues/7062 - /run/shm isn't writable or executable
# in trusty containers
- export MTR_MEM=/tmp
- env DEB_BUILD_OPTIONS="parallel=6" debian/autobake-deb.sh;
- ccache --show-stats
+ # Until OSX becomes a bit more stable: MDEV-12435
+ allow_failures:
+ - os: osx
+ compiler: clang
+ env: GCC_VERSION=4.8 TYPE=RelWithDebInfo MYSQL_TEST_SUITES=rpl
+ - os: osx
+ compiler: clang
+ env: GCC_VERSION=5 TYPE=RelWithDebInfo MYSQL_TEST_SUITES=main,archive,optimizer_unfixed_bugs,parts,sys_vars,unit,vcol,innodb,innodb_gis,innodb_zip,innodb_fts
+ - os: osx
+ compiler: clang
+ env: GCC_VERSION=6 TYPE=RelWithDebInfo MYSQL_TEST_SUITES=binlog,binlog_encryption,encryption,rocksdb
+ - os: osx
+ compiler: clang
+ env: GCC_VERSION=6 TYPE=RelWithDebInfo MYSQL_TEST_SUITES=csv,federated,funcs_1,funcs_2,gcol,handler,heap,json,maria,percona,perfschema,plugins,multi_source,roles
# Matrix include for coverity
# - env:
@@ -85,29 +136,26 @@ addons:
apt:
sources:
- ubuntu-toolchain-r-test
- - llvm-toolchain-trusty
- - llvm-toolchain-trusty-3.9
- llvm-toolchain-trusty-4.0
- packages: # make sure these match debian/control contents
+ - sourceline: 'deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-5.0 main'
+ packages: # make sure these match the build requirements
- gcc-5
- g++-5
- gcc-6
- g++-6
- - clang-3.9
- - llvm-3.9-dev
- clang-4.0
- llvm-4.0-dev
+ - clang-5.0
+ - llvm-5.0-dev
- libasan0
- bison
- chrpath
- cmake
- - debhelper
- - dh-apparmor
- - dpatch
- gdb
- libaio-dev
- libboost-dev
- libcurl3-dev
+ - libdbd-mysql
- libjudy-dev
- libncurses5-dev
- libpam0g-dev
@@ -119,16 +167,15 @@ addons:
- libxml2-dev
- lsb-release
- perl
- - po-debconf
- psmisc
- zlib1g-dev
- libcrack2-dev
+ - cracklib-runtime
- libjemalloc-dev
- libsnappy-dev
- liblzma-dev
- libzmq-dev
- - libdistro-info-perl
- - devscripts # implicit for any build on Ubuntu
+ - uuid-dev
# libsystemd-daemon-dev # https://github.com/travis-ci/apt-package-whitelist/issues/3882
diff --git a/BUILD/FINISH.sh b/BUILD/FINISH.sh
index b41c5c5139d..20784c49f0b 100644
--- a/BUILD/FINISH.sh
+++ b/BUILD/FINISH.sh
@@ -35,6 +35,11 @@ fi
# git clean -fdX removes all ignored (build) files
commands="\
git clean -fdX
+cd ./libmariadb
+git submodule update
+cd ../storage/rocksdb/rocksdb
+git submodule update
+cd ../../..
path=`dirname $0`
. \"$path/autorun.sh\""
diff --git a/BUILD/SETUP.sh b/BUILD/SETUP.sh
index 22357ce7fc8..2d6548dda0e 100755
--- a/BUILD/SETUP.sh
+++ b/BUILD/SETUP.sh
@@ -205,7 +205,7 @@ fi
max_no_embedded_configs="$SSL_LIBRARY --with-plugins=max"
max_no_qc_configs="$SSL_LIBRARY --with-plugins=max --without-query-cache"
-max_configs="$SSL_LIBRARY --with-plugins=max --with-embedded-server --with-libevent --without-plugin=plugin_file_key_management"
+max_configs="$SSL_LIBRARY --with-plugins=max --with-embedded-server --with-libevent --without-plugin=plugin_file_key_management --with-plugin-rocksdb=dynamic"
all_configs="$SSL_LIBRARY --with-plugins=max --with-embedded-server --with-innodb_plugin --with-libevent"
#
@@ -289,6 +289,11 @@ gcov_compile_flags="-fprofile-arcs -ftest-coverage"
gcov_compile_flags="$gcov_compile_flags -DDISABLE_TAO_ASM"
gcov_compile_flags="$gcov_compile_flags -DMYSQL_SERVER_SUFFIX=-gcov -DHAVE_gcov"
+#
+# The following plugins doesn't work on 32 bit systems
+disable_64_bit_plugins="--without-plugin-tokudb --without-plugin-rocksdb"
+
+
# GCC4 needs -fprofile-arcs -ftest-coverage on the linker command line (as well
# as on the compiler command line), and this requires setting LDFLAGS for BDB.
diff --git a/BUILD/compile-amd64-debug-max-no-ndb b/BUILD/compile-amd64-debug-max-no-ndb
deleted file mode 100755
index d45a51eba54..00000000000
--- a/BUILD/compile-amd64-debug-max-no-ndb
+++ /dev/null
@@ -1,25 +0,0 @@
-#! /bin/sh
-
-# Copyright (C) 2005, 2006 MySQL AB
-# Use is subject to license terms
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Library General Public
-# License as published by the Free Software Foundation; version 2
-# of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Library General Public License for more details.
-#
-# You should have received a copy of the GNU Library General Public
-# License along with this library; if not, write to the Free
-# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-# MA 02110-1301, USA
-
-path=`dirname $0`
-. "$path/SETUP.sh"
-extra_flags="$amd64_cflags $debug_cflags"
-extra_configs="$amd64_configs $debug_configs $max_configs"
-
-. "$path/FINISH.sh"
diff --git a/BUILD/compile-amd64-gprof-no-ndb b/BUILD/compile-amd64-gprof-no-ndb
deleted file mode 100755
index ef684274658..00000000000
--- a/BUILD/compile-amd64-gprof-no-ndb
+++ /dev/null
@@ -1,7 +0,0 @@
-#! /bin/sh
-path=`dirname $0`
-. "$path/SETUP.sh"
-extra_flags="$amd64_cflags -pg -g"
-extra_configs="$amd64_configs $max_configs --disable-shared $static_link"
-
-. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium b/BUILD/compile-pentium
index c197d9b49bf..af8e5b53a30 100755
--- a/BUILD/compile-pentium
+++ b/BUILD/compile-pentium
@@ -20,7 +20,7 @@ path=`dirname $0`
. "$path/SETUP.sh"
extra_flags="$pentium_cflags $fast_cflags"
-extra_configs="$pentium_configs"
+extra_configs="$pentium_configs $disable_64_bit_plugins"
strip=yes
. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium-debug b/BUILD/compile-pentium-debug
index faeb1b89597..ecbe04087c3 100755
--- a/BUILD/compile-pentium-debug
+++ b/BUILD/compile-pentium-debug
@@ -19,6 +19,6 @@ path=`dirname $0`
. "$path/SETUP.sh"
extra_flags="$pentium_cflags $debug_cflags"
-extra_configs="$pentium_configs $debug_configs"
+extra_configs="$pentium_configs $debug_configs $disable_64_bit_plugins"
. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium-debug-all b/BUILD/compile-pentium-debug-all
index 710ce8af63c..9ed5bf6b2cd 100755
--- a/BUILD/compile-pentium-debug-all
+++ b/BUILD/compile-pentium-debug-all
@@ -5,6 +5,6 @@ set -- "$@" --with-debug=full
. "$path/SETUP.sh"
extra_flags="$pentium_cflags $debug_cflags"
-extra_configs="$pentium_configs $debug_configs $all_configs $error_inject --with-experimental-collations"
+extra_configs="$pentium_configs $debug_configs $all_configs $error_inject --with-experimental-collations $disable_64_bit_plugins"
. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium-debug-max b/BUILD/compile-pentium-debug-max
index 0c925d8426f..68784eb34ea 100755
--- a/BUILD/compile-pentium-debug-max
+++ b/BUILD/compile-pentium-debug-max
@@ -19,6 +19,6 @@ path=`dirname $0`
. "$path/SETUP.sh"
extra_flags="$pentium_cflags $debug_cflags"
-extra_configs="$pentium_configs $debug_configs $max_configs $error_inject --with-experimental-collations"
+extra_configs="$pentium_configs $debug_configs $max_configs $error_inject --with-experimental-collations $disable_64_bit_plugins"
. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium-debug-max-no-embedded b/BUILD/compile-pentium-debug-max-no-embedded
index 2394c8aa2c7..d8bc896f89c 100755
--- a/BUILD/compile-pentium-debug-max-no-embedded
+++ b/BUILD/compile-pentium-debug-max-no-embedded
@@ -20,6 +20,6 @@ path=`dirname $0`
. "$path/SETUP.sh"
extra_flags="$pentium_cflags $debug_cflags"
-extra_configs="$pentium_configs $debug_configs $max_no_embedded_configs"
+extra_configs="$pentium_configs $debug_configs $max_no_embedded_configs $disable_64_bit_plugins"
. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium-debug-max-no-ndb b/BUILD/compile-pentium-debug-max-no-ndb
deleted file mode 100755
index 705164c20bc..00000000000
--- a/BUILD/compile-pentium-debug-max-no-ndb
+++ /dev/null
@@ -1,24 +0,0 @@
-#! /bin/sh
-
-# Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
-
-path=`dirname $0`
-. "$path/SETUP.sh"
-
-extra_flags="$pentium_cflags $debug_cflags"
-extra_configs="$pentium_configs $debug_configs $max_configs"
-
-. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium-debug-max-no-qc b/BUILD/compile-pentium-debug-max-no-qc
deleted file mode 100755
index 6407b4b09ad..00000000000
--- a/BUILD/compile-pentium-debug-max-no-qc
+++ /dev/null
@@ -1,10 +0,0 @@
-#! /bin/sh
-# Builds server without query cache support
-
-path=`dirname $0`
-. "$path/SETUP.sh"
-
-extra_flags="$pentium_cflags $debug_cflags"
-extra_configs="$pentium_configs $debug_configs $max_no_qc_configs"
-
-. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium-debug-openssl b/BUILD/compile-pentium-debug-openssl
index abf6b41a2d2..697b937bc93 100755
--- a/BUILD/compile-pentium-debug-openssl
+++ b/BUILD/compile-pentium-debug-openssl
@@ -19,7 +19,7 @@ path=`dirname $0`
. "$path/SETUP.sh"
extra_flags="$pentium_cflags $debug_cflags"
-extra_configs="$pentium_configs $debug_configs"
+extra_configs="$pentium_configs $debug_configs $disable_64_bit_plugins"
extra_configs="$extra_configs --with-debug --with-ssl=/usr"
diff --git a/BUILD/compile-pentium-debug-wsrep b/BUILD/compile-pentium-debug-wsrep
index ee68e3fd0c1..6528ed77f95 100644
--- a/BUILD/compile-pentium-debug-wsrep
+++ b/BUILD/compile-pentium-debug-wsrep
@@ -7,6 +7,6 @@ set -- "$@" --with-debug=full
extra_flags="$pentium_cflags $debug_cflags -g -O0 $wsrep_cflags"
c_warnings="$c_warnings $debug_extra_warnings"
cxx_warnings="$cxx_warnings $debug_extra_warnings"
-extra_configs="$pentium_configs $debug_configs $wsrep_configs --with-wsrep"
+extra_configs="$pentium_configs $debug_configs $wsrep_configs --with-wsrep $disable_64_bit_plugins"
. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium-gcov b/BUILD/compile-pentium-gcov
index 56072f619e5..9c5a61e9089 100755
--- a/BUILD/compile-pentium-gcov
+++ b/BUILD/compile-pentium-gcov
@@ -40,6 +40,6 @@ export LDFLAGS="$gcov_link_flags"
extra_flags="$pentium_cflags $debug_cflags $max_cflags $gcov_compile_flags"
c_warnings="$c_warnings $debug_extra_warnings"
cxx_warnings="$cxx_warnings $debug_extra_warnings"
-extra_configs="$pentium_configs $debug_configs $gcov_configs $max_configs"
+extra_configs="$pentium_configs $debug_configs $gcov_configs $max_configs $disable_64_bit_plugins"
. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium-gprof b/BUILD/compile-pentium-gprof
index 52231e7832a..de014e3ae8b 100755
--- a/BUILD/compile-pentium-gprof
+++ b/BUILD/compile-pentium-gprof
@@ -19,6 +19,6 @@ path=`dirname $0`
. "$path/SETUP.sh"
extra_flags="$pentium_cflags $gprof_compile_flags"
-extra_configs="$pentium_configs $debug_configs $gprof_link_flags"
+extra_configs="$pentium_configs $debug_configs $gprof_link_flags $disable_64_bit_plugins"
. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium-icc b/BUILD/compile-pentium-icc
index 23333a13c15..4fdc37b0667 100755
--- a/BUILD/compile-pentium-icc
+++ b/BUILD/compile-pentium-icc
@@ -35,6 +35,6 @@ extra_flags="$fast_cflags -unroll2 -ip -mp -restrict"
# icpc: error: problem during multi-file optimization compilation (code 1)
extra_flags="$extra_flags -no-ipo"
base_cxxflags="-fno-exceptions -fno-rtti"
-extra_configs="$pentium_configs $static_link"
+extra_configs="$pentium_configs $static_link $disable_64_bit_plugins"
. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium-max b/BUILD/compile-pentium-max
index 470596f8eb9..d2a61de8e13 100755
--- a/BUILD/compile-pentium-max
+++ b/BUILD/compile-pentium-max
@@ -20,6 +20,6 @@ path=`dirname $0`
. "$path/SETUP.sh"
extra_flags="$pentium_cflags $fast_cflags -g"
-extra_configs="$pentium_configs $max_configs"
+extra_configs="$pentium_configs $max_configs $disable_64_bit_plugins"
. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium-valgrind-max b/BUILD/compile-pentium-valgrind-max
index c4cb1845ba1..4f21e3574ff 100755
--- a/BUILD/compile-pentium-valgrind-max
+++ b/BUILD/compile-pentium-valgrind-max
@@ -33,6 +33,6 @@ path=`dirname $0`
. "$path/SETUP.sh"
extra_flags="$pentium_cflags $debug_cflags $valgrind_flags"
-extra_configs="$pentium_configs $debug_configs $valgrind_configs $max_configs"
+extra_configs="$pentium_configs $debug_configs $valgrind_configs $max_configs $disable_64_bit_plugins"
. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium-valgrind-max-no-ndb b/BUILD/compile-pentium-valgrind-max-no-ndb
deleted file mode 100755
index 4eb47cb2fe2..00000000000
--- a/BUILD/compile-pentium-valgrind-max-no-ndb
+++ /dev/null
@@ -1,26 +0,0 @@
-#! /bin/sh
-
-# Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Library General Public
-# License as published by the Free Software Foundation; version 2
-# of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Library General Public License for more details.
-#
-# You should have received a copy of the GNU Library General Public
-# License along with this library; if not, write to the Free
-# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-# MA 02110-1301, USA
-
-path=`dirname $0`
-. "$path/SETUP.sh"
-
-extra_flags="$pentium_cflags $debug_cflags $valgrind_flags"
-extra_configs="$pentium_configs $debug_configs $valgrind_configs $max_configs"
-
-. "$path/FINISH.sh"
diff --git a/BUILD/compile-pentium-wsrep b/BUILD/compile-pentium-wsrep
index eeb14310e4e..b0b8e408370 100644
--- a/BUILD/compile-pentium-wsrep
+++ b/BUILD/compile-pentium-wsrep
@@ -4,7 +4,7 @@ path=`dirname $0`
. "$path/SETUP.sh"
extra_flags="$pentium_cflags $fast_cflags $wsrep_cflags"
-extra_configs="$pentium_configs $wsrep_configs --with-wsrep"
+extra_configs="$pentium_configs $wsrep_configs --with-wsrep $disable_64_bit_plugins"
#strip=yes
diff --git a/BUILD/compile-ppc-debug-max-no-ndb b/BUILD/compile-ppc-debug-max-no-ndb
deleted file mode 100755
index 0642ddf7a57..00000000000
--- a/BUILD/compile-ppc-debug-max-no-ndb
+++ /dev/null
@@ -1,25 +0,0 @@
-#! /bin/sh
-
-# Copyright (c) 2005, 2006 MySQL AB
-# Use is subject to license terms
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
-
-path=`dirname $0`
-. "$path/SETUP.sh"
-
-extra_flags="$ppc_cflags $debug_cflags"
-extra_configs="$debug_configs $max_configs"
-
-. "$path/FINISH.sh"
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5c4e0037588..b0cf68351b7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,5 +1,5 @@
-# Copyright (c) 2006, 2014, Oracle and/or its affiliates.
-# Copyright (c) 2008, 2014, Monty Program Ab
+# Copyright (c) 2006, 2017, Oracle and/or its affiliates.
+# Copyright (c) 2008, 2017, MariaDB
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -220,11 +220,9 @@ IF(SECURITY_HARDENED)
MY_CHECK_AND_SET_COMPILER_FLAG("-D_FORTIFY_SOURCE=2" RELEASE RELWITHDEBINFO)
ENDIF()
-OPTION(ENABLE_DEBUG_SYNC "Enable debug sync (debug builds only)" ON)
-IF(ENABLE_DEBUG_SYNC)
- SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DENABLED_DEBUG_SYNC")
- SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DENABLED_DEBUG_SYNC")
-ENDIF()
+# Always enable debug sync for debug builds.
+SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DENABLED_DEBUG_SYNC")
+SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DENABLED_DEBUG_SYNC")
OPTION(ENABLE_GCOV "Enable gcov (debug, Linux builds only)" OFF)
IF (ENABLE_GCOV)
@@ -365,7 +363,7 @@ ENDIF()
SET (MYSQLD_STATIC_PLUGIN_LIBS "" CACHE INTERNAL "")
-# mariadb_connector_c fetches submodules which is useful for plugins
+INCLUDE(submodules)
INCLUDE(mariadb_connector_c) # this does ADD_SUBDIRECTORY(libmariadb)
# Add storage engines and plugins.
@@ -462,7 +460,6 @@ INSTALL_DOCUMENTATION(README.md CREDITS COPYING COPYING.thirdparty
# ${CMAKE_BINARY_DIR}/Docs/INFO_BIN)
IF(UNIX)
- INSTALL_DOCUMENTATION(Docs/INSTALL-BINARY COMPONENT Readme)
INSTALL_DOCUMENTATION(Docs/INSTALL-BINARY Docs/README-wsrep COMPONENT Readme)
ENDIF()
diff --git a/CREDITS b/CREDITS
index d352232ad2e..6288c2cdea4 100644
--- a/CREDITS
+++ b/CREDITS
@@ -4,17 +4,19 @@ organization registered in the USA.
The current main sponsors of the MariaDB Foundation are:
Alibaba Cloud https://intl.aliyun.com (2017)
-Booking.com https://www.booking.com (2013 - 2017)
-Development Bank of Singapore https://dbs.com (2016 - 2017)
-MariaDB Corporation https://www.mariadb.com (2013 - 2017)
-Visma https://visma.com (2015 - 2017)
-Acronis http://acronis.com (2016 - 2017)
-Nexedi https://www.nexedi.com (2016 - 2017)
-Automattic https://automattic.com (2014 - 2017)
-Tencent Game DBA http://tencentdba.com/about (2016 - 2017)
-Tencent TDSQL http://tdsql.org/ (2016 - 2017)
-Verkkokauppa.com https://www.verkkokauppa.com (2015 - 2017)
-Virtuozzo https://virtuozzo.com (2016 - 2017)
+Booking.com https://www.booking.com (2013)
+Tencent Cloud https://cloud.tencent.com (2017)
+Development Bank of Singapore https://dbs.com (2016)
+IBM https://www.ibm.com (2017)
+MariaDB Corporation https://www.mariadb.com (2013)
+Visma https://visma.com (2015)
+Acronis http://acronis.com (2016)
+Nexedi https://www.nexedi.com (2016)
+Automattic https://automattic.com (2014)
+Tencent Game DBA http://tencentdba.com/about (2016)
+Tencent TDSQL http://tdsql.org (2016)
+Verkkokauppa.com https://www.verkkokauppa.com (2015)
+Virtuozzo https://virtuozzo.com (2016)
For a full list of sponsors, see
https://mariadb.org/about/supporters/
diff --git a/README.md b/README.md
index f34e6a43b71..58e08f89ca0 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@ Please read the CREDITS file for details about the MariaDB Foundation,
and who is developing MariaDB.
MariaDB is developed by many of the original developers of MySQL who
-now work for the MariadB Foundation and the MariaDB Corporation, and by many people in
+now work for the MariaDB Foundation and the MariaDB Corporation, and by many people in
the community.
MySQL, which is the base of MariaDB, is a product and trademark of Oracle
diff --git a/VERSION b/VERSION
index ed825f4c577..b816ecbfe54 100644
--- a/VERSION
+++ b/VERSION
@@ -1,3 +1,3 @@
MYSQL_VERSION_MAJOR=10
MYSQL_VERSION_MINOR=2
-MYSQL_VERSION_PATCH=6
+MYSQL_VERSION_PATCH=9
diff --git a/client/client_priv.h b/client/client_priv.h
index e96e187fb34..ba1a1fddfae 100644
--- a/client/client_priv.h
+++ b/client/client_priv.h
@@ -120,4 +120,3 @@ enum options_client
Name of the performance schema database.
*/
#define PERFORMANCE_SCHEMA_DB_NAME "performance_schema"
-
diff --git a/client/mysql.cc b/client/mysql.cc
index 69924010a15..fb9ffbef669 100644
--- a/client/mysql.cc
+++ b/client/mysql.cc
@@ -1,6 +1,6 @@
/*
Copyright (c) 2000, 2014, Oracle and/or its affiliates.
- Copyright (c) 2009, 2016, MariaDB
+ Copyright (c) 2009, 2017, MariaDB
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -149,7 +149,7 @@ static my_bool ignore_errors=0,wait_flag=0,quick=0,
default_pager_set= 0, opt_sigint_ignore= 0,
auto_vertical_output= 0,
show_warnings= 0, executing_query= 0,
- ignore_spaces= 0, opt_progress_reports;
+ ignore_spaces= 0, opt_binhex= 0, opt_progress_reports;
static my_bool debug_info_flag, debug_check_flag, batch_abort_on_error;
static my_bool column_types_flag;
static my_bool preserve_comments= 0;
@@ -1145,8 +1145,6 @@ int main(int argc,char *argv[])
outfile[0]=0; // no (default) outfile
strmov(pager, "stdout"); // the default, if --pager wasn't given
- mysql_init(&mysql);
-
{
char *tmp=getenv("PAGER");
if (tmp && strlen(tmp))
@@ -1185,7 +1183,11 @@ int main(int argc,char *argv[])
}
defaults_argv=argv;
if ((status.exit_status= get_options(argc, (char **) argv)))
- mysql_end(-1);
+ {
+ free_defaults(defaults_argv);
+ my_end(0);
+ exit(status.exit_status);
+ }
if (status.batch && !status.line_buff &&
!(status.line_buff= batch_readline_init(MAX_BATCH_BUFFER_SIZE, stdin)))
@@ -1494,6 +1496,8 @@ static struct my_option my_long_options[] =
{"batch", 'B',
"Don't use history file. Disable interactive behavior. (Enables --silent.)",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
+ {"binary-as-hex", 'b', "Print binary data as hex", &opt_binhex, &opt_binhex,
+ 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"character-sets-dir", OPT_CHARSETS_DIR,
"Directory for character set files.", &charsets_dir,
&charsets_dir, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
@@ -2321,8 +2325,10 @@ static bool add_line(String &buffer, char *line, ulong line_length,
continue;
}
#endif
- if (!*ml_comment && inchar == '\\' &&
- !(*in_string &&
+ if (!*ml_comment && inchar == '\\' && *in_string != '`' &&
+ !(*in_string == '"' &&
+ (mysql.server_status & SERVER_STATUS_ANSI_QUOTES)) &&
+ !(*in_string &&
(mysql.server_status & SERVER_STATUS_NO_BACKSLASH_ESCAPES)))
{
// Found possbile one character command like \c
@@ -3061,7 +3067,6 @@ static int com_server_help(String *buffer __attribute__((unused)),
{
unsigned int num_fields= mysql_num_fields(result);
my_ulonglong num_rows= mysql_num_rows(result);
- mysql_fetch_fields(result);
if (num_fields==3 && num_rows==1)
{
if (!(cur= mysql_fetch_row(result)))
@@ -3315,7 +3320,8 @@ com_go(String *buffer,char *line __attribute__((unused)))
print_table_data_html(result);
else if (opt_xml)
print_table_data_xml(result);
- else if (vertical || (auto_vertical_output && (terminal_width < get_result_width(result))))
+ else if (vertical || (auto_vertical_output &&
+ (terminal_width < get_result_width(result))))
print_table_data_vertically(result);
else if (opt_silent && verbose <= 2 && !output_tables)
print_tab_data(result);
@@ -3533,6 +3539,41 @@ print_field_types(MYSQL_RES *result)
}
+/* Used to determine if we should invoke print_as_hex for this field */
+
+static bool
+is_binary_field(MYSQL_FIELD *field)
+{
+ if ((field->charsetnr == 63) &&
+ (field->type == MYSQL_TYPE_BIT ||
+ field->type == MYSQL_TYPE_BLOB ||
+ field->type == MYSQL_TYPE_LONG_BLOB ||
+ field->type == MYSQL_TYPE_MEDIUM_BLOB ||
+ field->type == MYSQL_TYPE_TINY_BLOB ||
+ field->type == MYSQL_TYPE_VAR_STRING ||
+ field->type == MYSQL_TYPE_STRING ||
+ field->type == MYSQL_TYPE_VARCHAR ||
+ field->type == MYSQL_TYPE_GEOMETRY))
+ return 1;
+ return 0;
+}
+
+
+/* Print binary value as hex literal (0x ...) */
+
+static void
+print_as_hex(FILE *output_file, const char *str, ulong len, ulong total_bytes_to_send)
+{
+ const char *ptr= str, *end= ptr+len;
+ ulong i;
+ fprintf(output_file, "0x");
+ for(; ptr < end; ptr++)
+ fprintf(output_file, "%02X", *((uchar*)ptr));
+ for (i= 2*len+2; i < total_bytes_to_send; i++)
+ tee_putc((int)' ', output_file);
+}
+
+
static void
print_table_data(MYSQL_RES *result)
{
@@ -3559,6 +3600,8 @@ print_table_data(MYSQL_RES *result)
length= MY_MAX(length,field->max_length);
if (length < 4 && !IS_NOT_NULL(field->flags))
length=4; // Room for "NULL"
+ if (opt_binhex && is_binary_field(field))
+ length= 2 + length * 2;
field->max_length=length;
num_flag[mysql_field_tell(result) - 1]= IS_NUM(field->type);
separator.fill(separator.length()+length+2,'-');
@@ -3626,9 +3669,11 @@ print_table_data(MYSQL_RES *result)
many extra padding-characters we should send with the printing function.
*/
visible_length= charset_info->cset->numcells(charset_info, buffer, buffer + data_length);
- extra_padding= data_length - visible_length;
+ extra_padding= (uint) (data_length - visible_length);
- if (field_max_length > MAX_COLUMN_LENGTH)
+ if (opt_binhex && is_binary_field(field))
+ print_as_hex(PAGER, cur[off], lengths[off], field_max_length);
+ else if (field_max_length > MAX_COLUMN_LENGTH)
tee_print_sized_data(buffer, data_length, MAX_COLUMN_LENGTH+extra_padding, FALSE);
else
{
@@ -3762,11 +3807,15 @@ print_table_data_html(MYSQL_RES *result)
if (interrupted_query)
break;
ulong *lengths=mysql_fetch_lengths(result);
+ field= mysql_fetch_fields(result);
(void) tee_fputs("
", PAGER);
for (uint i=0; i < mysql_num_fields(result); i++)
{
(void) tee_fputs("", PAGER);
- xmlencode_print(cur[i], lengths[i]);
+ if (opt_binhex && is_binary_field(&field[i]))
+ print_as_hex(PAGER, cur[i], lengths[i], lengths[i]);
+ else
+ xmlencode_print(cur[i], lengths[i]);
(void) tee_fputs(" | ", PAGER);
}
(void) tee_fputs("
", PAGER);
@@ -3802,7 +3851,10 @@ print_table_data_xml(MYSQL_RES *result)
if (cur[i])
{
tee_fprintf(PAGER, "\">");
- xmlencode_print(cur[i], lengths[i]);
+ if (opt_binhex && is_binary_field(&fields[i]))
+ print_as_hex(PAGER, cur[i], lengths[i], lengths[i]);
+ else
+ xmlencode_print(cur[i], lengths[i]);
tee_fprintf(PAGER, "\n");
}
else
@@ -3849,13 +3901,19 @@ print_table_data_vertically(MYSQL_RES *result)
{
unsigned int i;
const char *p;
-
+ if (opt_binhex && is_binary_field(field))
+ fprintf(PAGER, "0x");
for (i= 0, p= cur[off]; i < lengths[off]; i+= 1, p+= 1)
{
- if (*p == '\0')
- tee_putc((int)' ', PAGER);
+ if (opt_binhex && is_binary_field(field))
+ fprintf(PAGER, "%02X", *((uchar*)p));
else
- tee_putc((int)*p, PAGER);
+ {
+ if (*p == '\0')
+ tee_putc((int)' ', PAGER);
+ else
+ tee_putc((int)*p, PAGER);
+ }
}
tee_putc('\n', PAGER);
}
@@ -3865,7 +3923,6 @@ print_table_data_vertically(MYSQL_RES *result)
}
}
-
/* print_warnings should be called right after executing a statement */
static void print_warnings()
@@ -4002,11 +4059,19 @@ print_tab_data(MYSQL_RES *result)
while ((cur = mysql_fetch_row(result)))
{
lengths=mysql_fetch_lengths(result);
- safe_put_field(cur[0],lengths[0]);
+ field= mysql_fetch_fields(result);
+ if (opt_binhex && is_binary_field(&field[0]))
+ print_as_hex(PAGER, cur[0], lengths[0], lengths[0]);
+ else
+ safe_put_field(cur[0],lengths[0]);
+
for (uint off=1 ; off < mysql_num_fields(result); off++)
{
(void) tee_fputs("\t", PAGER);
- safe_put_field(cur[off], lengths[off]);
+ if (opt_binhex && field && is_binary_field(&field[off]))
+ print_as_hex(PAGER, cur[off], lengths[off], lengths[off]);
+ else
+ safe_put_field(cur[off], lengths[off]);
}
(void) tee_fputs("\n", PAGER);
}
@@ -4802,10 +4867,11 @@ com_status(String *buffer __attribute__((unused)),
tee_fprintf(stdout, "Protocol:\t\tCompressed\n");
#endif
- if ((status_str= mysql_stat(&mysql)) && !mysql_error(&mysql)[0])
+ const char *pos;
+ if ((status_str= mysql_stat(&mysql)) && !mysql_error(&mysql)[0] &&
+ (pos= strchr(status_str,' ')))
{
ulong sec;
- const char *pos= strchr(status_str,' ');
/* print label */
tee_fprintf(stdout, "%.*s\t\t\t", (int) (pos-status_str), status_str);
if ((status_str= str2int(pos,10,0,LONG_MAX,(long*) &sec)))
diff --git a/client/mysql_upgrade.c b/client/mysql_upgrade.c
index ee63fda2e39..c10dc20e7d7 100644
--- a/client/mysql_upgrade.c
+++ b/client/mysql_upgrade.c
@@ -1,6 +1,6 @@
/*
Copyright (c) 2006, 2013, Oracle and/or its affiliates.
- Copyright (c) 2010, 2016, MariaDB
+ Copyright (c) 2010, 2017, MariaDB
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -1130,7 +1130,7 @@ static int check_version_match(void)
int main(int argc, char **argv)
{
- char self_name[FN_REFLEN];
+ char self_name[FN_REFLEN + 1];
MY_INIT(argv[0]);
@@ -1138,7 +1138,7 @@ int main(int argc, char **argv)
if (GetModuleFileName(NULL, self_name, FN_REFLEN) == 0)
#endif
{
- strncpy(self_name, argv[0], FN_REFLEN);
+ strmake_buf(self_name, argv[0]);
}
if (init_dynamic_string(&ds_args, "", 512, 256) ||
@@ -1171,6 +1171,8 @@ int main(int argc, char **argv)
{
int fd= create_temp_file(cnf_file_path, opt_tmpdir[0] ? opt_tmpdir : NULL,
"mysql_upgrade-", O_CREAT | O_WRONLY, MYF(MY_FAE));
+ if (fd < 0)
+ die(NULL);
my_write(fd, USTRING_WITH_LEN( "[client]\n"), MYF(MY_FAE));
my_write(fd, (uchar*)ds_args.str, ds_args.length, MYF(MY_FAE));
my_close(fd, MYF(0));
diff --git a/client/mysqladmin.cc b/client/mysqladmin.cc
index 7491458c161..45b332a6cd6 100644
--- a/client/mysqladmin.cc
+++ b/client/mysqladmin.cc
@@ -1,6 +1,6 @@
/*
Copyright (c) 2000, 2014, Oracle and/or its affiliates.
- Copyright (c) 2010, 2016, MariaDB
+ Copyright (c) 2010, 2017, MariaDB
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -47,6 +47,7 @@ static uint opt_count_iterations= 0, my_end_arg;
static ulong opt_connect_timeout, opt_shutdown_timeout;
static char * unix_port=0;
static char *opt_plugin_dir= 0, *opt_default_auth= 0;
+static bool sql_log_bin_off= false;
#ifdef HAVE_SMEM
static char *shared_memory_base_name=0;
@@ -600,6 +601,31 @@ static my_bool sql_connect(MYSQL *mysql, uint wait)
}
+static int maybe_disable_binlog(MYSQL *mysql)
+{
+ if (opt_local && !sql_log_bin_off)
+ {
+ if (mysql_query(mysql, "set local sql_log_bin=0"))
+ {
+ my_printf_error(0, "SET LOCAL SQL_LOG_BIN=0 failed; error: '%-.200s'",
+ error_flags, mysql_error(mysql));
+ return -1;
+ }
+ }
+ sql_log_bin_off= true;
+ return 0;
+}
+
+
+int flush(MYSQL *mysql, const char *what)
+{
+ char buf[FN_REFLEN];
+ my_snprintf(buf, sizeof(buf), "flush %s%s",
+ (opt_local && !sql_log_bin_off ? "local " : ""), what);
+ return mysql_query(mysql, buf);
+}
+
+
/**
@brief Execute all commands
@@ -616,6 +642,7 @@ static my_bool sql_connect(MYSQL *mysql, uint wait)
static int execute_commands(MYSQL *mysql,int argc, char **argv)
{
+ int ret = 0;
const char *status;
/*
MySQL documentation relies on the fact that mysqladmin will
@@ -628,17 +655,6 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
struct my_rnd_struct rand_st;
char buff[FN_REFLEN + 20];
- if (opt_local)
- {
- sprintf(buff, "set local sql_log_bin=0");
- if (mysql_query(mysql, buff))
- {
- my_printf_error(0, "SET LOCAL SQL_LOG_BIN=0 failed; error: '%-.200s'",
- error_flags, mysql_error(mysql));
- return -1;
- }
- }
-
for (; argc > 0 ; argv++,argc--)
{
int command;
@@ -650,6 +666,8 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
my_printf_error(0, "Too few arguments to create", error_flags);
return 1;
}
+ if (maybe_disable_binlog(mysql))
+ return -1;
sprintf(buff,"create database `%.*s`",FN_REFLEN,argv[1]);
if (mysql_query(mysql,buff))
{
@@ -667,6 +685,8 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
my_printf_error(0, "Too few arguments to drop", error_flags);
return 1;
}
+ if (maybe_disable_binlog(mysql))
+ return -1;
if (drop_db(mysql,argv[1]))
return -1;
argc--; argv++;
@@ -707,7 +727,7 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
}
case ADMIN_FLUSH_PRIVILEGES:
case ADMIN_RELOAD:
- if (mysql_query(mysql,"flush privileges"))
+ if (flush(mysql, "privileges"))
{
my_printf_error(0, "reload failed; error: '%s'", error_flags,
mysql_error(mysql));
@@ -911,7 +931,7 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
}
case ADMIN_FLUSH_LOGS:
{
- if (mysql_query(mysql,"flush logs"))
+ if (flush(mysql, "logs"))
{
my_printf_error(0, "flush failed; error: '%s'", error_flags,
mysql_error(mysql));
@@ -921,7 +941,7 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
}
case ADMIN_FLUSH_BINARY_LOG:
{
- if (mysql_query(mysql, "flush binary logs"))
+ if (flush(mysql, "binary logs"))
{
my_printf_error(0, "flush failed; error: '%s'", error_flags,
mysql_error(mysql));
@@ -931,7 +951,7 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
}
case ADMIN_FLUSH_ENGINE_LOG:
{
- if (mysql_query(mysql,"flush engine logs"))
+ if (flush(mysql, "engine logs"))
{
my_printf_error(0, "flush failed; error: '%s'", error_flags,
mysql_error(mysql));
@@ -941,7 +961,7 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
}
case ADMIN_FLUSH_ERROR_LOG:
{
- if (mysql_query(mysql, "flush error logs"))
+ if (flush(mysql, "error logs"))
{
my_printf_error(0, "flush failed; error: '%s'", error_flags,
mysql_error(mysql));
@@ -951,7 +971,7 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
}
case ADMIN_FLUSH_GENERAL_LOG:
{
- if (mysql_query(mysql, "flush general logs"))
+ if (flush(mysql, "general logs"))
{
my_printf_error(0, "flush failed; error: '%s'", error_flags,
mysql_error(mysql));
@@ -961,7 +981,7 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
}
case ADMIN_FLUSH_RELAY_LOG:
{
- if (mysql_query(mysql, "flush relay logs"))
+ if (flush(mysql, "relay logs"))
{
my_printf_error(0, "flush failed; error: '%s'", error_flags,
mysql_error(mysql));
@@ -971,7 +991,7 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
}
case ADMIN_FLUSH_SLOW_LOG:
{
- if (mysql_query(mysql,"flush slow logs"))
+ if (flush(mysql, "slow logs"))
{
my_printf_error(0, "flush failed; error: '%s'", error_flags,
mysql_error(mysql));
@@ -981,7 +1001,7 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
}
case ADMIN_FLUSH_HOSTS:
{
- if (mysql_query(mysql,"flush hosts"))
+ if (flush(mysql, "hosts"))
{
my_printf_error(0, "flush failed; error: '%s'", error_flags,
mysql_error(mysql));
@@ -991,7 +1011,7 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
}
case ADMIN_FLUSH_TABLES:
{
- if (mysql_query(mysql,"flush tables"))
+ if (flush(mysql, "tables"))
{
my_printf_error(0, "flush failed; error: '%s'", error_flags,
mysql_error(mysql));
@@ -1001,7 +1021,7 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
}
case ADMIN_FLUSH_STATUS:
{
- if (mysql_query(mysql,"flush status"))
+ if (flush(mysql, "status"))
{
my_printf_error(0, "flush failed; error: '%s'", error_flags,
mysql_error(mysql));
@@ -1011,7 +1031,7 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
}
case ADMIN_FLUSH_TABLE_STATISTICS:
{
- if (mysql_query(mysql,"flush table_statistics"))
+ if (flush(mysql, "table_statistics"))
{
my_printf_error(0, "flush failed; error: '%s'", error_flags,
mysql_error(mysql));
@@ -1021,7 +1041,7 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
}
case ADMIN_FLUSH_INDEX_STATISTICS:
{
- if (mysql_query(mysql,"flush index_statistics"))
+ if (flush(mysql, "index_statistics"))
{
my_printf_error(0, "flush failed; error: '%s'", error_flags,
mysql_error(mysql));
@@ -1031,7 +1051,7 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
}
case ADMIN_FLUSH_USER_STATISTICS:
{
- if (mysql_query(mysql,"flush user_statistics"))
+ if (flush(mysql, "user_statistics"))
{
my_printf_error(0, "flush failed; error: '%s'", error_flags,
mysql_error(mysql));
@@ -1041,7 +1061,7 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
}
case ADMIN_FLUSH_USER_RESOURCES:
{
- if (mysql_query(mysql, "flush user_resources"))
+ if (flush(mysql, "user_resources"))
{
my_printf_error(0, "flush failed; error: '%s'", error_flags,
mysql_error(mysql));
@@ -1051,7 +1071,7 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
}
case ADMIN_FLUSH_CLIENT_STATISTICS:
{
- if (mysql_query(mysql,"flush client_statistics"))
+ if (flush(mysql, "client_statistics"))
{
my_printf_error(0, "flush failed; error: '%s'", error_flags,
mysql_error(mysql));
@@ -1061,9 +1081,8 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
}
case ADMIN_FLUSH_ALL_STATISTICS:
{
- if (mysql_query(mysql,
- "flush table_statistics,index_statistics,"
- "user_statistics,client_statistics"))
+ if (flush(mysql, "table_statistics,index_statistics,"
+ "user_statistics,client_statistics"))
{
my_printf_error(0, "flush failed; error: '%s'", error_flags,
mysql_error(mysql));
@@ -1073,9 +1092,8 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
}
case ADMIN_FLUSH_ALL_STATUS:
{
- if (mysql_query(mysql,
- "flush status,table_statistics,index_statistics,"
- "user_statistics,client_statistics"))
+ if (flush(mysql, "status,table_statistics,index_statistics,"
+ "user_statistics,client_statistics"))
{
my_printf_error(0, "flush failed; error: '%s'", error_flags,
mysql_error(mysql));
@@ -1093,6 +1111,8 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
start_time=time((time_t*) 0);
my_rnd_init(&rand_st,(ulong) start_time,(ulong) start_time/2);
+ if (maybe_disable_binlog(mysql))
+ return -1;
if (argc < 1)
{
my_printf_error(0, "Too few arguments to change password", error_flags);
@@ -1106,7 +1126,8 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
if (strcmp(typed_password, verified) != 0)
{
my_printf_error(0,"Passwords don't match",MYF(ME_BELL));
- return -1;
+ ret = -1;
+ goto password_done;
}
}
else
@@ -1133,7 +1154,8 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
{
my_printf_error(0, "Could not determine old_passwords setting from server; error: '%s'",
error_flags, mysql_error(mysql));
- return -1;
+ ret = -1;
+ goto password_done;
}
else
{
@@ -1144,7 +1166,8 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
"Could not get old_passwords setting from "
"server; error: '%s'",
error_flags, mysql_error(mysql));
- return -1;
+ ret = -1;
+ goto password_done;
}
if (!mysql_num_rows(res))
old= 1;
@@ -1169,15 +1192,15 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
{
my_printf_error(0, "Can't turn off logging; error: '%s'",
error_flags, mysql_error(mysql));
- return -1;
+ ret = -1;
}
+ else
if (mysql_query(mysql,buff))
{
if (mysql_errno(mysql)!=1290)
{
my_printf_error(0,"unable to change password; error: '%s'",
error_flags, mysql_error(mysql));
- return -1;
}
else
{
@@ -1191,9 +1214,10 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
" --skip-grant-tables).\n"
"Use: \"mysqladmin flush-privileges password '*'\""
" instead", error_flags);
- return -1;
}
+ ret = -1;
}
+password_done:
/* free up memory from prompted password */
if (typed_password != argv[1])
{
@@ -1300,7 +1324,7 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
return 1;
}
}
- return 0;
+ return ret;
}
/**
diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc
index 92534501fb1..584f6955453 100644
--- a/client/mysqlbinlog.cc
+++ b/client/mysqlbinlog.cc
@@ -68,6 +68,7 @@ CHARSET_INFO* system_charset_info= &my_charset_utf8_general_ci;
/* Needed for Flashback */
DYNAMIC_ARRAY binlog_events; // Storing the events output string
+DYNAMIC_ARRAY events_in_stmt; // Storing the events that in one statement
String stop_event_string; // Storing the STOP_EVENT output string
char server_version[SERVER_VERSION_LENGTH];
@@ -894,6 +895,25 @@ static bool print_row_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev,
print_event_info->m_table_map_ignored.get_table(table_id);
bool skip_event= (ignored_map != NULL);
+ if (opt_flashback)
+ {
+ Rows_log_event *e= (Rows_log_event*) ev;
+ // The last Row_log_event will be the first event in Flashback
+ if (is_stmt_end)
+ e->clear_flags(Rows_log_event::STMT_END_F);
+ // The first Row_log_event will be the last event in Flashback
+ if (events_in_stmt.elements == 0)
+ e->set_flags(Rows_log_event::STMT_END_F);
+ // Update the temp_buf
+ e->update_flags();
+
+ if (insert_dynamic(&events_in_stmt, (uchar *) &ev))
+ {
+ error("Out of memory: can't allocate memory to store the flashback events.");
+ exit(1);
+ }
+ }
+
/*
end of statement check:
i) destroy/free ignored maps
@@ -945,7 +965,36 @@ static bool print_row_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev,
if (skip_event)
return 0;
- return print_base64(print_event_info, ev);
+ if (!opt_flashback)
+ return print_base64(print_event_info, ev);
+ else
+ {
+ if (is_stmt_end)
+ {
+ bool res= false;
+ Log_event *e= NULL;
+
+ // Print the row_event from the last one to the first one
+ for (uint i= events_in_stmt.elements; i > 0; --i)
+ {
+ e= *(dynamic_element(&events_in_stmt, i - 1, Log_event**));
+ res= res || print_base64(print_event_info, e);
+ }
+ // Copy all output into the Log_event
+ ev->output_buf.copy(e->output_buf);
+ // Delete Log_event
+ for (uint i= 0; i < events_in_stmt.elements-1; ++i)
+ {
+ e= *(dynamic_element(&events_in_stmt, i, Log_event**));
+ delete e;
+ }
+ reset_dynamic(&events_in_stmt);
+
+ return res;
+ }
+ }
+
+ return 0;
}
@@ -1386,6 +1435,8 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev,
}
if (print_base64(print_event_info, ev))
goto err;
+ if (opt_flashback)
+ reset_dynamic(&events_in_stmt);
break;
}
case WRITE_ROWS_EVENT:
@@ -1402,9 +1453,12 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev,
case DELETE_ROWS_COMPRESSED_EVENT_V1:
{
Rows_log_event *e= (Rows_log_event*) ev;
+ bool is_stmt_end= e->get_flags(Rows_log_event::STMT_END_F);
if (print_row_event(print_event_info, ev, e->get_table_id(),
e->get_flags(Rows_log_event::STMT_END_F)))
goto err;
+ if (!is_stmt_end)
+ destroy_evt= FALSE;
break;
}
case PRE_GA_WRITE_ROWS_EVENT:
@@ -1412,9 +1466,12 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev,
case PRE_GA_UPDATE_ROWS_EVENT:
{
Old_rows_log_event *e= (Old_rows_log_event*) ev;
+ bool is_stmt_end= e->get_flags(Rows_log_event::STMT_END_F);
if (print_row_event(print_event_info, ev, e->get_table_id(),
e->get_flags(Old_rows_log_event::STMT_END_F)))
goto err;
+ if (!is_stmt_end)
+ destroy_evt= FALSE;
break;
}
case START_ENCRYPTION_EVENT:
@@ -1459,7 +1516,7 @@ end:
&my_charset_bin);
else
{
- if (push_dynamic(&binlog_events, (uchar *) &tmp_str))
+ if (insert_dynamic(&binlog_events, (uchar *) &tmp_str))
{
error("Out of memory: can't allocate memory to store the flashback events.");
exit(1);
@@ -2480,7 +2537,7 @@ static Exit_status dump_remote_log_entries(PRINT_EVENT_INFO *print_event_info,
int2store(buf + BIN_LOG_HEADER_SIZE, binlog_flags);
size_t tlen = strlen(logname);
- if (tlen > UINT_MAX)
+ if (tlen > sizeof(buf) - 10)
{
error("Log name too long.");
DBUG_RETURN(ERROR_STOP);
@@ -2915,9 +2972,12 @@ int main(int argc, char** argv)
my_set_max_open_files(open_files_limit);
if (opt_flashback)
+ {
my_init_dynamic_array(&binlog_events, sizeof(LEX_STRING), 1024, 1024,
MYF(0));
-
+ my_init_dynamic_array(&events_in_stmt, sizeof(Rows_log_event*), 1024, 1024,
+ MYF(0));
+ }
if (opt_stop_never)
to_last_remote_log= TRUE;
@@ -3031,6 +3091,7 @@ int main(int argc, char** argv)
}
fprintf(result_file, "COMMIT\n/*!*/;\n");
delete_dynamic(&binlog_events);
+ delete_dynamic(&events_in_stmt);
}
/* Set delimiter back to semicolon */
diff --git a/client/mysqlcheck.c b/client/mysqlcheck.c
index c1ea002515b..42785b6dc7c 100644
--- a/client/mysqlcheck.c
+++ b/client/mysqlcheck.c
@@ -1,6 +1,6 @@
/*
Copyright (c) 2001, 2013, Oracle and/or its affiliates.
- Copyright (c) 2010, 2016, MariaDB
+ Copyright (c) 2010, 2017, MariaDB
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
diff --git a/client/mysqldump.c b/client/mysqldump.c
index 281a7a6c7f2..c8ebd2217a3 100644
--- a/client/mysqldump.c
+++ b/client/mysqldump.c
@@ -92,8 +92,7 @@
static void add_load_option(DYNAMIC_STRING *str, const char *option,
const char *option_value);
-static ulong find_set(TYPELIB *lib, const char *x, size_t length,
- char **err_pos, uint *err_len);
+static ulong find_set(TYPELIB *, const char *, size_t, char **, uint *);
static char *alloc_query_str(ulong size);
static void field_escape(DYNAMIC_STRING* in, const char *from);
@@ -114,7 +113,8 @@ static my_bool verbose= 0, opt_no_create_info= 0, opt_no_data= 0, opt_no_data_m
opt_slave_apply= 0,
opt_include_master_host_port= 0,
opt_events= 0, opt_comments_used= 0,
- opt_alltspcs=0, opt_notspcs= 0, opt_logging;
+ opt_alltspcs=0, opt_notspcs= 0, opt_logging,
+ opt_drop_trigger= 0 ;
static my_bool insert_pat_inited= 0, debug_info_flag= 0, debug_check_flag= 0;
static ulong opt_max_allowed_packet, opt_net_buffer_length;
static MYSQL mysql_connection,*mysql=0;
@@ -233,6 +233,9 @@ static struct my_option my_long_options[] =
{"add-drop-table", OPT_DROP, "Add a DROP TABLE before each create.",
&opt_drop, &opt_drop, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0,
0},
+ {"add-drop-trigger", 0, "Add a DROP TRIGGER before each create.",
+ &opt_drop_trigger, &opt_drop_trigger, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0,
+ 0},
{"add-locks", OPT_LOCKS, "Add locks around INSERT statements.",
&opt_lock, &opt_lock, 0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0,
0},
@@ -2170,6 +2173,7 @@ static void print_xml_comment(FILE *xml_file, size_t len,
case '-':
if (*(comment_string + 1) == '-') /* Only one hyphen allowed. */
break;
+ /* fall through */
default:
fputc(*comment_string, xml_file);
break;
@@ -2207,7 +2211,6 @@ static void print_comment(FILE *sql_file, my_bool is_error, const char *format,
print_xml_comment(sql_file, strlen(comment_buff), comment_buff);
}
-
/*
create_delimiter
Generate a new (null-terminated) string that does not exist in query
@@ -2552,7 +2555,7 @@ static uint dump_routines_for_db(char *db)
query_buff);
print_comment(sql_file, 1,
"-- does %s have permissions on mysql.proc?\n\n",
- current_user);
+ fix_for_comment(current_user));
maybe_die(EX_MYSQLERR,"%s has insufficent privileges to %s!",
current_user, query_buff);
}
@@ -2847,6 +2850,8 @@ static uint get_table_structure(char *table, char *db, char *table_type,
my_free(scv_buff);
+ if (path)
+ my_fclose(sql_file, MYF(MY_WME));
DBUG_RETURN(0);
}
else
@@ -3281,6 +3286,10 @@ static void dump_trigger_old(FILE *sql_file, MYSQL_RES *show_triggers_rs,
if (opt_compact)
fprintf(sql_file, "/*!50003 SET @OLD_SQL_MODE=@@SQL_MODE*/;\n");
+ if (opt_drop_trigger)
+ fprintf(sql_file, "/*!50032 DROP TRIGGER IF EXISTS %s */;\n",
+ (*show_trigger_row)[0]);
+
fprintf(sql_file,
"DELIMITER ;;\n"
"/*!50003 SET SESSION SQL_MODE=\"%s\" */;;\n"
@@ -3361,6 +3370,10 @@ static int dump_trigger(FILE *sql_file, MYSQL_RES *show_create_trigger_rs,
switch_sql_mode(sql_file, ";", row[1]);
+ if (opt_drop_trigger)
+ fprintf(sql_file, "/*!50032 DROP TRIGGER IF EXISTS %s */;\n",
+ row[0]);
+
query_str= cover_definer_clause(row[2], strlen(row[2]),
C_STRING_WITH_LEN("50017"),
C_STRING_WITH_LEN("50003"),
@@ -5471,7 +5484,7 @@ static ulong find_set(TYPELIB *lib, const char *x, size_t length,
var_len= (uint) (pos - start);
strmake(buff, start, MY_MIN(sizeof(buff) - 1, var_len));
find= find_type(buff, lib, FIND_TYPE_BASIC);
- if (!find)
+ if (find <= 0)
{
*err_pos= (char*) start;
*err_len= var_len;
@@ -5929,8 +5942,7 @@ static my_bool get_view_structure(char *table, char* db)
dynstr_free(&ds_view);
}
- if (switch_character_set_results(mysql, default_charset))
- DBUG_RETURN(1);
+ switch_character_set_results(mysql, default_charset);
/* If a separate .sql file was opened, close it now */
if (sql_file != md_result_file)
diff --git a/client/mysqlimport.c b/client/mysqlimport.c
index 688789ec436..38b2eb5f672 100644
--- a/client/mysqlimport.c
+++ b/client/mysqlimport.c
@@ -1,6 +1,6 @@
/*
Copyright (c) 2000, 2015, Oracle and/or its affiliates.
- Copyright (c) 2011, 2016, MariaDB
+ Copyright (c) 2011, 2017, MariaDB
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -677,7 +677,7 @@ int main(int argc, char **argv)
MYF(0))))
return -2;
- for (counter= 0; *argv != NULL; argv++) /* Loop through tables */
+ for (; *argv != NULL; argv++) /* Loop through tables */
{
pthread_mutex_lock(&counter_mutex);
while (counter == opt_use_threads)
diff --git a/client/mysqlshow.c b/client/mysqlshow.c
index e3e30abc426..46494311f7d 100644
--- a/client/mysqlshow.c
+++ b/client/mysqlshow.c
@@ -1,6 +1,6 @@
/*
Copyright (c) 2000, 2015, Oracle and/or its affiliates.
- Copyright (c) 2010, 2016, MariaDB
+ Copyright (c) 2010, 2017, MariaDB
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
diff --git a/client/mysqlslap.c b/client/mysqlslap.c
index b3229980e77..a78bf35d51b 100644
--- a/client/mysqlslap.c
+++ b/client/mysqlslap.c
@@ -1,6 +1,6 @@
/*
Copyright (c) 2005, 2015, Oracle and/or its affiliates.
- Copyright (c) 2010, 2016, MariaDB
+ Copyright (c) 2010, 2017, MariaDB
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
diff --git a/client/mysqltest.cc b/client/mysqltest.cc
index d930369e303..05113f02a44 100644
--- a/client/mysqltest.cc
+++ b/client/mysqltest.cc
@@ -1,5 +1,5 @@
/* Copyright (c) 2000, 2013, Oracle and/or its affiliates.
- Copyright (c) 2009, 2016, Monty Program Ab.
+ Copyright (c) 2009, 2017, MariaDB
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -82,7 +82,7 @@ static my_bool non_blocking_api_enabled= 0;
#define MAX_DELIMITER_LENGTH 16
#define DEFAULT_MAX_CONN 64
-#define DIE_BUFF_SIZE 8192
+#define DIE_BUFF_SIZE 256*1024
/* Flags controlling send and reap */
#define QUERY_SEND_FLAG 1
@@ -574,15 +574,17 @@ struct st_replace *glob_replace= 0;
void replace_strings_append(struct st_replace *rep, DYNAMIC_STRING* ds,
const char *from, int len);
-static void cleanup_and_exit(int exit_code) __attribute__((noreturn));
+ATTRIBUTE_NORETURN
+static void cleanup_and_exit(int exit_code);
-void really_die(const char *msg) __attribute__((noreturn));
+ATTRIBUTE_NORETURN
+void really_die(const char *msg);
void report_or_die(const char *fmt, ...) ATTRIBUTE_FORMAT(printf, 1, 2);
-void die(const char *fmt, ...) ATTRIBUTE_FORMAT(printf, 1, 2)
- __attribute__((noreturn));
+ATTRIBUTE_NORETURN ATTRIBUTE_FORMAT(printf, 1, 2)
+void die(const char *fmt, ...);
static void make_error_message(char *buf, size_t len, const char *fmt, va_list args);
-void abort_not_supported_test(const char *fmt, ...) ATTRIBUTE_FORMAT(printf, 1, 2)
- __attribute__((noreturn));
+ATTRIBUTE_NORETURN ATTRIBUTE_FORMAT(printf, 1, 2)
+void abort_not_supported_test(const char *fmt, ...);
void verbose_msg(const char *fmt, ...) ATTRIBUTE_FORMAT(printf, 1, 2);
void log_msg(const char *fmt, ...) ATTRIBUTE_FORMAT(printf, 1, 2);
@@ -602,7 +604,7 @@ void do_eval(DYNAMIC_STRING *query_eval, const char *query,
void str_to_file(const char *fname, char *str, int size);
void str_to_file2(const char *fname, char *str, int size, my_bool append);
-void fix_win_paths(char *val, int len);
+void fix_win_paths(char *val, size_t len);
const char *get_errname_from_code (uint error_code);
int multi_reg_replace(struct st_replace_regex* r,char* val);
@@ -1063,7 +1065,7 @@ void do_eval(DYNAMIC_STRING *query_eval, const char *query,
if (!(v= var_get(p, &p, 0, 0)))
{
report_or_die( "Bad variable in eval");
- return;
+ DBUG_VOID_RETURN;
}
dynstr_append_mem(query_eval, v->str_val, v->str_val_len);
}
@@ -1721,12 +1723,22 @@ int cat_file(DYNAMIC_STRING* ds, const char* filename)
{
int fd;
size_t len;
- char buff[16384];
+ char *buff;
if ((fd= my_open(filename, O_RDONLY, MYF(0))) < 0)
return 1;
- while((len= my_read(fd, (uchar*)&buff,
- sizeof(buff)-1, MYF(0))) > 0)
+
+ len= (size_t) my_seek(fd, 0, SEEK_END, MYF(0));
+ my_seek(fd, 0, SEEK_SET, MYF(0));
+ if (len == (size_t)MY_FILEPOS_ERROR ||
+ !(buff= (char*)my_malloc(len + 1, MYF(0))))
+ {
+ my_close(fd, MYF(0));
+ return 1;
+ }
+ len= my_read(fd, (uchar*)buff, len, MYF(0));
+ my_close(fd, MYF(0));
+
{
char *p= buff, *start= buff,*end=buff+len;
while (p < end)
@@ -1749,7 +1761,7 @@ int cat_file(DYNAMIC_STRING* ds, const char* filename)
*p= 0;
replace_dynstr_append_mem(ds, start, p-start);
}
- my_close(fd, MYF(0));
+ my_free(buff);
return 0;
}
@@ -1776,7 +1788,7 @@ static int run_command(char* cmd,
if (!(res_file= popen(cmd, "r")))
{
report_or_die("popen(\"%s\", \"r\") failed", cmd);
- return -1;
+ DBUG_RETURN(-1);
}
while (fgets(buf, sizeof(buf), res_file))
@@ -2642,6 +2654,7 @@ void var_query_set(VAR *var, const char *query, const char** query_end)
if (!mysql)
{
struct st_command command;
+ DBUG_ASSERT(query_end);
memset(&command, 0, sizeof(command));
command.query= (char*)query;
command.first_word_len= (*query_end - query);
@@ -2873,7 +2886,7 @@ void var_set_query_get_value(struct st_command *command, VAR *var)
dynstr_free(&ds_query);
dynstr_free(&ds_col);
eval_expr(var, "", 0);
- return;
+ DBUG_VOID_RETURN;
}
{
@@ -2898,7 +2911,7 @@ void var_set_query_get_value(struct st_command *command, VAR *var)
ds_col.str, ds_query.str);
dynstr_free(&ds_query);
dynstr_free(&ds_col);
- return;
+ DBUG_VOID_RETURN;
}
DBUG_PRINT("info", ("Found column %d with name '%s'",
i, fields[i].name));
@@ -3344,7 +3357,7 @@ void do_exec(struct st_command *command)
if (!*cmd)
{
report_or_die("Missing argument in exec");
- return;
+ DBUG_VOID_RETURN;
}
command->last_argument= command->end;
@@ -3370,6 +3383,12 @@ void do_exec(struct st_command *command)
#endif
#endif
+ if (disable_result_log)
+ {
+ /* Collect stderr output as well, for the case app. crashes or returns error.*/
+ dynstr_append(&ds_cmd, " 2>&1");
+ }
+
DBUG_PRINT("info", ("Executing '%s' as '%s'",
command->first_argument, ds_cmd.str));
@@ -3378,7 +3397,7 @@ void do_exec(struct st_command *command)
dynstr_free(&ds_cmd);
if (command->abort_on_error)
report_or_die("popen(\"%s\", \"r\") failed", command->first_argument);
- return;
+ DBUG_VOID_RETURN;
}
ds_result= &ds_res;
@@ -3405,16 +3424,7 @@ void do_exec(struct st_command *command)
len--;
}
#endif
- if (disable_result_log)
- {
- if (len)
- buf[len-1] = 0;
- DBUG_PRINT("exec_result",("%s", buf));
- }
- else
- {
- replace_dynstr_append_mem(ds_result, buf, len);
- }
+ replace_dynstr_append_mem(ds_result, buf, len);
}
error= pclose(res_file);
@@ -3424,7 +3434,7 @@ void do_exec(struct st_command *command)
dynstr_free(&ds_sorted);
}
- if (error > 0)
+ if (error)
{
uint status= WEXITSTATUS(error);
int i;
@@ -3436,7 +3446,7 @@ void do_exec(struct st_command *command)
ds_cmd.str, error, status, errno,
ds_res.str);
dynstr_free(&ds_cmd);
- return;
+ DBUG_VOID_RETURN;
}
DBUG_PRINT("info",
@@ -3470,6 +3480,12 @@ void do_exec(struct st_command *command)
}
dynstr_free(&ds_cmd);
+
+ if (disable_result_log)
+ {
+ /* Disable output in case of successful exit.*/
+ dynstr_set(&ds_res,"");
+ }
DBUG_VOID_RETURN;
}
@@ -3569,7 +3585,7 @@ void do_system(struct st_command *command)
if (strlen(command->first_argument) == 0)
{
report_or_die("Missing arguments to system, nothing to do!");
- return;
+ DBUG_VOID_RETURN;
}
init_dynamic_string(&ds_cmd, 0, command->query_len + 64, 256);
@@ -3607,6 +3623,37 @@ void do_system(struct st_command *command)
}
+/* returns TRUE if path is inside a sandbox */
+bool is_sub_path(const char *path, size_t plen, const char *sandbox)
+{
+ size_t len= strlen(sandbox);
+ if (!sandbox || !len || plen <= len || memcmp(path, sandbox, len - 1)
+ || path[len] != '/')
+ return false;
+ return true;
+}
+
+
+/* returns TRUE if path cannot be modified */
+bool bad_path(const char *path)
+{
+ size_t plen= strlen(path);
+
+ const char *vardir= getenv("MYSQLTEST_VARDIR");
+ if (is_sub_path(path, plen, vardir))
+ return false;
+
+ const char *tmpdir= getenv("MYSQL_TMP_DIR");
+ if (is_sub_path(path, plen, tmpdir))
+ return false;
+
+ report_or_die("Path '%s' is not a subdirectory of MYSQLTEST_VARDIR '%s'"
+ "or MYSQL_TMP_DIR '%s'",
+ path, vardir, tmpdir);
+ return true;
+}
+
+
/*
SYNOPSIS
set_wild_chars
@@ -3665,6 +3712,9 @@ void do_remove_file(struct st_command *command)
rm_args, sizeof(rm_args)/sizeof(struct command_arg),
' ');
+ if (bad_path(ds_filename.str))
+ DBUG_VOID_RETURN;
+
DBUG_PRINT("info", ("removing file: %s", ds_filename.str));
error= my_delete(ds_filename.str, MYF(disable_warnings ? 0 : MY_WME)) != 0;
handle_command_error(command, error, my_errno);
@@ -3708,6 +3758,9 @@ void do_remove_files_wildcard(struct st_command *command)
' ');
fn_format(dirname, ds_directory.str, "", "", MY_UNPACK_FILENAME);
+ if (bad_path(ds_directory.str))
+ DBUG_VOID_RETURN;
+
DBUG_PRINT("info", ("listing directory: %s", dirname));
if (!(dir_info= my_dir(dirname, MYF(MY_DONT_SORT | MY_WANT_STAT | MY_WME))))
{
@@ -3782,6 +3835,9 @@ void do_copy_file(struct st_command *command)
sizeof(copy_file_args)/sizeof(struct command_arg),
' ');
+ if (bad_path(ds_to_file.str))
+ DBUG_VOID_RETURN;
+
DBUG_PRINT("info", ("Copy %s to %s", ds_from_file.str, ds_to_file.str));
/* MY_HOLD_ORIGINAL_MODES prevents attempts to chown the file */
error= (my_copy(ds_from_file.str, ds_to_file.str,
@@ -3819,6 +3875,9 @@ void do_move_file(struct st_command *command)
sizeof(move_file_args)/sizeof(struct command_arg),
' ');
+ if (bad_path(ds_to_file.str))
+ DBUG_VOID_RETURN;
+
DBUG_PRINT("info", ("Move %s to %s", ds_from_file.str, ds_to_file.str));
error= (my_rename(ds_from_file.str, ds_to_file.str,
MYF(disable_warnings ? 0 : MY_WME)) != 0);
@@ -3857,6 +3916,9 @@ void do_chmod_file(struct st_command *command)
sizeof(chmod_file_args)/sizeof(struct command_arg),
' ');
+ if (bad_path(ds_file.str))
+ DBUG_VOID_RETURN;
+
/* Parse what mode to set */
if (ds_mode.length != 4 ||
str2int(ds_mode.str, 8, 0, INT_MAX, &mode) == NullS)
@@ -3928,6 +3990,9 @@ void do_mkdir(struct st_command *command)
mkdir_args, sizeof(mkdir_args)/sizeof(struct command_arg),
' ');
+ if (bad_path(ds_dirname.str))
+ DBUG_VOID_RETURN;
+
DBUG_PRINT("info", ("creating directory: %s", ds_dirname.str));
error= my_mkdir(ds_dirname.str, 0777, MYF(MY_WME)) != 0;
handle_command_error(command, error, my_errno);
@@ -3935,6 +4000,65 @@ void do_mkdir(struct st_command *command)
DBUG_VOID_RETURN;
}
+
+/*
+ Remove directory recursively.
+*/
+static int rmtree(const char *dir)
+{
+ char path[FN_REFLEN];
+ char sep[]={ FN_LIBCHAR, 0 };
+ int err=0;
+
+ MY_DIR *dir_info= my_dir(dir, MYF(MY_DONT_SORT | MY_WANT_STAT));
+ if (!dir_info)
+ return 1;
+
+ for (uint i= 0; i < dir_info->number_of_files; i++)
+ {
+ FILEINFO *file= dir_info->dir_entry + i;
+ /* Skip "." and ".." */
+ if (!strcmp(file->name, ".") || !strcmp(file->name, ".."))
+ continue;
+
+ strxnmov(path, sizeof(path), dir, sep, file->name, NULL);
+
+ if (!MY_S_ISDIR(file->mystat->st_mode))
+ {
+ err= my_delete(path, 0);
+#ifdef _WIN32
+ /*
+ On Windows, check and possible reset readonly attribute.
+ my_delete(), or DeleteFile does not remove theses files.
+ */
+ if (err)
+ {
+ DWORD attr= GetFileAttributes(path);
+ if (attr != INVALID_FILE_ATTRIBUTES &&
+ (attr & FILE_ATTRIBUTE_READONLY))
+ {
+ SetFileAttributes(path, attr &~ FILE_ATTRIBUTE_READONLY);
+ err= my_delete(path, 0);
+ }
+ }
+#endif
+ }
+ else
+ err= rmtree(path);
+
+ if(err)
+ break;
+ }
+
+ my_dirend(dir_info);
+
+ if (!err)
+ err= rmdir(dir);
+
+ return err;
+}
+
+
/*
SYNOPSIS
do_rmdir
@@ -3942,12 +4066,11 @@ void do_mkdir(struct st_command *command)
DESCRIPTION
rmdir
- Remove the empty directory
+ Remove the directory tree
*/
void do_rmdir(struct st_command *command)
{
- int error;
static DYNAMIC_STRING ds_dirname;
const struct command_arg rmdir_args[] = {
{ "dirname", ARG_STRING, TRUE, &ds_dirname, "Directory to remove" }
@@ -3958,9 +4081,13 @@ void do_rmdir(struct st_command *command)
rmdir_args, sizeof(rmdir_args)/sizeof(struct command_arg),
' ');
+ if (bad_path(ds_dirname.str))
+ DBUG_VOID_RETURN;
+
DBUG_PRINT("info", ("removing directory: %s", ds_dirname.str));
- error= rmdir(ds_dirname.str) != 0;
- handle_command_error(command, error, errno);
+ if (rmtree(ds_dirname.str))
+ handle_command_error(command, 1, errno);
+
dynstr_free(&ds_dirname);
DBUG_VOID_RETURN;
}
@@ -4073,6 +4200,9 @@ static void do_list_files_write_file_command(struct st_command *command,
list_files_args,
sizeof(list_files_args)/sizeof(struct command_arg), ' ');
+ if (bad_path(ds_filename.str))
+ DBUG_VOID_RETURN;
+
init_dynamic_string(&ds_content, "", 1024, 1024);
error= get_list_files(&ds_content, &ds_dirname, &ds_wild);
handle_command_error(command, error, my_errno);
@@ -4124,7 +4254,8 @@ void read_until_delimiter(DYNAMIC_STRING *ds,
while (1)
{
c= my_getc(cur_file->file);
-
+ if (c == '\r')
+ c= my_getc(cur_file->file);
if (c == '\n')
{
cur_file->lineno++;
@@ -4175,6 +4306,9 @@ void do_write_file_command(struct st_command *command, my_bool append)
sizeof(write_file_args)/sizeof(struct command_arg),
' ');
+ if (bad_path(ds_filename.str))
+ DBUG_VOID_RETURN;
+
if (!append && access(ds_filename.str, F_OK) == 0)
{
/* The file should not be overwritten */
@@ -4556,7 +4690,7 @@ void do_perl(struct st_command *command)
if (command->abort_on_error)
die("popen(\"%s\", \"r\") failed", buf);
dynstr_free(&ds_delimiter);
- return;
+ DBUG_VOID_RETURN;
}
while (fgets(buf, sizeof(buf), res_file))
@@ -6388,6 +6522,16 @@ my_bool end_of_query(int c)
}
+static inline bool is_escape_char(char c, char in_string)
+{
+ if (c != '\\' || in_string == '`') return false;
+ if (!cur_con) return true;
+ uint server_status= cur_con->mysql->server_status;
+ if (server_status & SERVER_STATUS_NO_BACKSLASH_ESCAPES) return false;
+ return !(server_status & SERVER_STATUS_ANSI_QUOTES && in_string == '"');
+}
+
+
/*
Read one "line" from the file
@@ -6414,7 +6558,7 @@ my_bool end_of_query(int c)
int read_line(char *buf, int size)
{
- char c, UNINIT_VAR(last_quote), last_char= 0;
+ char c, last_quote=0, last_char= 0;
char *p= buf, *buf_end= buf + size - 1;
int skip_char= 0;
my_bool have_slash= FALSE;
@@ -6496,7 +6640,7 @@ int read_line(char *buf, int size)
state= R_Q;
}
}
- have_slash= (c == '\\');
+ have_slash= is_escape_char(c, last_quote);
break;
case R_COMMENT:
@@ -6566,7 +6710,7 @@ int read_line(char *buf, int size)
case R_Q:
if (c == last_quote)
state= R_NORMAL;
- else if (c == '\\')
+ else if (is_escape_char(c, last_quote))
state= R_SLASH_IN_Q;
break;
@@ -7345,7 +7489,7 @@ void free_win_path_patterns()
=> all \ from c:\mysql\m... until next space is converted into /
*/
-void fix_win_paths(char *val, int len)
+void fix_win_paths(char *val, size_t len)
{
#ifdef _WIN32
uint i;
@@ -9875,25 +10019,39 @@ bool parse_re_part(char *start_re, char *end_re,
Returns: st_replace_regex struct with pairs of substitutions
*/
+void append_replace_regex(char*, char*, struct st_replace_regex*, char**);
struct st_replace_regex* init_replace_regex(char* expr)
{
+ char *expr_end, *buf_p;
struct st_replace_regex* res;
- char* buf,*expr_end;
- char* p, start_re, end_re= 1;
- char* buf_p;
uint expr_len= strlen(expr);
- struct st_regex reg;
/* my_malloc() will die on fail with MY_FAE */
res=(struct st_replace_regex*)my_malloc(
- sizeof(*res)+expr_len ,MYF(MY_FAE+MY_WME));
+ sizeof(*res)+8192 ,MYF(MY_FAE+MY_WME));
my_init_dynamic_array(&res->regex_arr,sizeof(struct st_regex), 128, 128, MYF(0));
- buf= (char*)res + sizeof(*res);
expr_end= expr + expr_len;
+ buf_p= (char*)res + sizeof(*res);
+ append_replace_regex(expr, expr_end, res, &buf_p);
+
+ res->odd_buf_len= res->even_buf_len= 8192;
+ res->even_buf= (char*)my_malloc(res->even_buf_len,MYF(MY_WME+MY_FAE));
+ res->odd_buf= (char*)my_malloc(res->odd_buf_len,MYF(MY_WME+MY_FAE));
+ res->buf= res->even_buf;
+
+ return res;
+}
+
+
+void append_replace_regex(char* expr, char *expr_end, struct st_replace_regex* res,
+ char **buf_p)
+{
+ char* p, start_re, end_re= 1;
+ struct st_regex reg;
+
p= expr;
- buf_p= buf;
/* for each regexp substitution statement */
while (p < expr_end)
@@ -9912,13 +10070,34 @@ struct st_replace_regex* init_replace_regex(char* expr)
}
start_re= 0;
- reg.pattern= buf_p;
- if (parse_re_part(&start_re, &end_re, &p, expr_end, &buf_p))
- goto err;
+ reg.pattern= *buf_p;
- reg.replace= buf_p;
- if (parse_re_part(&start_re, &end_re, &p, expr_end, &buf_p))
- goto err;
+ /* Allow variable for the *entire* list of replacements */
+ if (*p == '$')
+ {
+ const char *v_end;
+ VAR *val= var_get(p, &v_end, 0, 1);
+
+ if (val)
+ {
+ char *expr, *expr_end;
+ expr= val->str_val;
+ expr_end= expr + val->str_val_len;
+ append_replace_regex(expr, expr_end, res, buf_p);
+ }
+
+ p= (char *) v_end + 1;
+ continue;
+ }
+ else
+ {
+ if (parse_re_part(&start_re, &end_re, &p, expr_end, buf_p))
+ goto err;
+
+ reg.replace= *buf_p;
+ if (parse_re_part(&start_re, &end_re, &p, expr_end, buf_p))
+ goto err;
+ }
/* Check if we should do matching case insensitive */
if (p < expr_end && *p == 'i')
@@ -9931,17 +10110,12 @@ struct st_replace_regex* init_replace_regex(char* expr)
if (insert_dynamic(&res->regex_arr, ®))
die("Out of memory");
}
- res->odd_buf_len= res->even_buf_len= 8192;
- res->even_buf= (char*)my_malloc(res->even_buf_len,MYF(MY_WME+MY_FAE));
- res->odd_buf= (char*)my_malloc(res->odd_buf_len,MYF(MY_WME+MY_FAE));
- res->buf= res->even_buf;
- return res;
+ return;
err:
my_free(res);
die("Error parsing replace_regex \"%s\"", expr);
- return 0;
}
/*
@@ -10021,12 +10195,6 @@ void do_get_replace_regex(struct st_command *command)
{
char *expr= command->first_argument;
free_replace_regex();
- /* Allow variable for the *entire* list of replacements */
- if (*expr == '$')
- {
- VAR *val= var_get(expr, NULL, 0, 1);
- expr= val ? val->str_val : NULL;
- }
if (expr && *expr && !(glob_replace_regex=init_replace_regex(expr)))
die("Could not init replace_regex");
command->last_argument= command->end;
diff --git a/cmake/build_configurations/mysql_release.cmake b/cmake/build_configurations/mysql_release.cmake
index 726a151a5b1..fcac8dde6e9 100644
--- a/cmake/build_configurations/mysql_release.cmake
+++ b/cmake/build_configurations/mysql_release.cmake
@@ -88,20 +88,24 @@ ENDIF()
OPTION(ENABLED_LOCAL_INFILE "" ON)
SET(WITH_INNODB_SNAPPY OFF CACHE STRING "")
IF(WIN32)
+ SET(WITH_LIBARCHIVE STATIC CACHE STRING "")
ELSEIF(RPM)
SET(WITH_SSL system CACHE STRING "")
SET(WITH_ZLIB system CACHE STRING "")
SET(CHECKMODULE /usr/bin/checkmodule CACHE STRING "")
SET(SEMODULE_PACKAGE /usr/bin/semodule_package CACHE STRING "")
+ SET(WITH_LIBARCHIVE ON CACHE STRING "")
ELSEIF(DEB)
SET(WITH_SSL system CACHE STRING "")
SET(WITH_ZLIB system CACHE STRING "")
SET(WITH_LIBWRAP ON)
SET(HAVE_EMBEDDED_PRIVILEGE_CONTROL ON)
+ SET(WITH_LIBARCHIVE ON CACHE STRING "")
ELSE()
SET(WITH_SSL bundled CACHE STRING "")
SET(WITH_ZLIB bundled CACHE STRING "")
SET(WITH_JEMALLOC static CACHE STRING "")
+ SET(WITH_LIBARCHIVE STATIC CACHE STRING "")
ENDIF()
IF(NOT COMPILATION_COMMENT)
diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake
index dced85f3a22..47daa88d0a2 100644
--- a/cmake/cpack_rpm.cmake
+++ b/cmake/cpack_rpm.cmake
@@ -23,13 +23,22 @@ SET(CPACK_COMPONENT_SHAREDLIBRARIES_GROUP "shared")
SET(CPACK_COMPONENT_COMMON_GROUP "common")
SET(CPACK_COMPONENT_CLIENTPLUGINS_GROUP "common")
SET(CPACK_COMPONENT_COMPAT_GROUP "compat")
+SET(CPACK_COMPONENT_BACKUP_GROUP "backup")
+
SET(CPACK_COMPONENTS_ALL Server ManPagesServer IniFiles Server_Scripts
SupportFiles Development ManPagesDevelopment
ManPagesTest Readme ManPagesClient Test
- Common Client SharedLibraries ClientPlugins)
+ Common Client SharedLibraries ClientPlugins
+ backup
+)
SET(CPACK_RPM_PACKAGE_NAME ${CPACK_PACKAGE_NAME})
-SET(CPACK_PACKAGE_FILE_NAME "${CPACK_RPM_PACKAGE_NAME}-${VERSION}-${RPM}-${CMAKE_SYSTEM_PROCESSOR}")
+IF(CMAKE_VERSION VERSION_LESS "3.6.0")
+ SET(CPACK_PACKAGE_FILE_NAME "${CPACK_RPM_PACKAGE_NAME}-${VERSION}-${RPM}-${CMAKE_SYSTEM_PROCESSOR}")
+ELSE()
+ SET(CPACK_RPM_FILE_NAME "RPM-DEFAULT")
+ SET(CPACK_RPM_DEBUGINFO_PACKAGE ON)
+ENDIF()
SET(CPACK_RPM_PACKAGE_RELEASE "1%{?dist}")
SET(CPACK_RPM_PACKAGE_LICENSE "GPLv2")
@@ -112,6 +121,7 @@ SET(CPACK_RPM_client_USER_FILELIST ${ignored} "%config(noreplace) ${INSTALL_SYSC
SET(CPACK_RPM_compat_USER_FILELIST ${ignored})
SET(CPACK_RPM_devel_USER_FILELIST ${ignored})
SET(CPACK_RPM_test_USER_FILELIST ${ignored})
+SET(CPACK_RPM_backup_USER_FILELIST ${ignored})
# "set/append array" - append a set of strings, separated by a space
MACRO(SETA var)
diff --git a/cmake/create_initial_db.cmake.in b/cmake/create_initial_db.cmake.in
index c77e28d882c..baa48847815 100644
--- a/cmake/create_initial_db.cmake.in
+++ b/cmake/create_initial_db.cmake.in
@@ -82,7 +82,3 @@ EXECUTE_PROCESS (
IF(NOT RESULT EQUAL 0)
MESSAGE(FATAL_ERROR "Could not create initial database \n ${OUT} \n ${ERR}")
ENDIF()
-
-EXECUTE_PROCESS (
- COMMAND "@CMAKE_COMMAND@" -E touch ${CMAKE_CURRENT_BINARY_DIR}/initdb.dep
-)
diff --git a/cmake/for_clients.cmake b/cmake/for_clients.cmake
index 636610e7fba..e5916c56ddc 100644
--- a/cmake/for_clients.cmake
+++ b/cmake/for_clients.cmake
@@ -45,7 +45,7 @@ MACRO(EXTRACT_LINK_LIBRARIES target var)
ENDIF()
ENDMACRO()
-EXTRACT_LINK_LIBRARIES(mariadb LIBS)
+EXTRACT_LINK_LIBRARIES(libmariadb LIBS)
EXTRACT_LINK_LIBRARIES(mysqlserver EMB_LIBS)
SET(LIBS "-lmariadb ${ZLIB_DEPS} ${LIBS} ${openssl_libs}")
@@ -72,6 +72,6 @@ REPLACE_FOR_CLIENTS(CFLAGS "[DU]DBUG_OFF" "[DU]SAFE_MUTEX" "[DU]NDEBUG"
"xstrconst" "xc99=none" "AC99" "restrict" "W[-A-Za-z]*=[-A-Za-z0-9]*")
# Same for --libs
-REPLACE_FOR_CLIENTS(LIBS lmtmalloc static-libcxa i-static static-intel)
+REPLACE_FOR_CLIENTS(LIBS "Wl,[^ ]*" lmtmalloc static-libcxa i-static static-intel)
REPLACE_FOR_CLIENTS(EMB_LIBS lmtmalloc static-libcxa i-static static-intel)
diff --git a/cmake/mariadb_connector_c.cmake b/cmake/mariadb_connector_c.cmake
index 400fcda0c1a..6773314313b 100644
--- a/cmake/mariadb_connector_c.cmake
+++ b/cmake/mariadb_connector_c.cmake
@@ -1,17 +1,3 @@
-IF(NOT EXISTS ${CMAKE_SOURCE_DIR}/libmariadb/CMakeLists.txt AND GIT_EXECUTABLE)
- EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule init
- WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
- EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule update
- WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
-ENDIF()
-IF(NOT EXISTS ${CMAKE_SOURCE_DIR}/libmariadb/CMakeLists.txt)
- MESSAGE(FATAL_ERROR "No MariaDB Connector/C! Run
- git submodule init
- git submodule update
-Then restart the build.
-")
-ENDIF()
-
SET(OPT CONC_)
IF (CMAKE_BUILD_TYPE STREQUAL "Debug")
diff --git a/cmake/os/Windows.cmake b/cmake/os/Windows.cmake
index 22f1ff7f30d..37fd204ed72 100644
--- a/cmake/os/Windows.cmake
+++ b/cmake/os/Windows.cmake
@@ -139,8 +139,8 @@ IF(MSVC)
ENDIF()
#TODO: update the code and remove the disabled warnings
- SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4800 /wd4805 /wd4996")
- SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4800 /wd4805 /wd4996 /wd4291 /wd4577 /we4099")
+ SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4800 /wd4805 /wd4996 /we4700")
+ SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4800 /wd4805 /wd4996 /wd4291 /wd4577 /we4099 /we4700")
ENDIF()
diff --git a/cmake/pcre.cmake b/cmake/pcre.cmake
index 45d9bc01ddb..894bde38974 100644
--- a/cmake/pcre.cmake
+++ b/cmake/pcre.cmake
@@ -5,7 +5,7 @@ MACRO (CHECK_PCRE)
IF(WITH_PCRE STREQUAL "system" OR WITH_PCRE STREQUAL "auto")
CHECK_LIBRARY_EXISTS(pcre pcre_stack_guard "" HAVE_PCRE)
ENDIF()
- IF(NOT HAVE_PCRE)
+ IF(NOT HAVE_PCRE OR WITH_PCRE STREQUAL "bundled")
IF (WITH_PCRE STREQUAL "system")
MESSAGE(FATAL_ERROR "system pcre is not found or unusable")
ENDIF()
diff --git a/cmake/plugin.cmake b/cmake/plugin.cmake
index 354ee53c7bb..df3115bc8f3 100644
--- a/cmake/plugin.cmake
+++ b/cmake/plugin.cmake
@@ -201,13 +201,15 @@ MACRO(MYSQL_ADD_PLUGIN)
# executable to the linker command line (it would result into link error).
# Thus we skip TARGET_LINK_LIBRARIES on Linux, as it would only generate
# an additional dependency.
- IF(NOT ARG_CLIENT)
+ IF(ARG_RECOMPILE_FOR_EMBEDDED OR ARG_STORAGE_ENGINE)
IF(MSVC)
ADD_DEPENDENCIES(${target} gen_mysqld_lib)
TARGET_LINK_LIBRARIES(${target} mysqld_import_lib)
ELSEIF(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux")
TARGET_LINK_LIBRARIES (${target} mysqld)
ENDIF()
+ ELSEIF(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND NOT WITH_ASAN)
+ TARGET_LINK_LIBRARIES (${target} "-Wl,--no-undefined")
ENDIF()
ADD_DEPENDENCIES(${target} GenError ${ARG_DEPENDENCIES})
@@ -233,9 +235,11 @@ MACRO(MYSQL_ADD_PLUGIN)
SET(CPACK_RPM_${ARG_COMPONENT}_PACKAGE_PROVIDES "cmake_bug_13248" PARENT_SCOPE)
SET(CPACK_RPM_${ARG_COMPONENT}_PACKAGE_OBSOLETES "cmake_bug_13248" PARENT_SCOPE)
SET(CPACK_RPM_${ARG_COMPONENT}_USER_FILELIST ${ignored} PARENT_SCOPE)
- IF(NOT ARG_CLIENT AND NOT ARG_CONFIG AND UNIX)
- SET(ARG_CONFIG "${CMAKE_CURRENT_BINARY_DIR}/${target}.cnf")
- FILE(WRITE ${ARG_CONFIG} "[mariadb]\nplugin-load-add=${ARG_MODULE_OUTPUT_NAME}.so\n")
+ IF(NOT ARG_CLIENT AND UNIX)
+ IF (NOT ARG_CONFIG)
+ SET(ARG_CONFIG "${CMAKE_CURRENT_BINARY_DIR}/${target}.cnf")
+ FILE(WRITE ${ARG_CONFIG} "[mariadb]\nplugin-load-add=${ARG_MODULE_OUTPUT_NAME}.so\n")
+ ENDIF()
INSTALL(FILES ${ARG_CONFIG} COMPONENT ${ARG_COMPONENT} DESTINATION ${INSTALL_SYSCONF2DIR})
SET(CPACK_RPM_${ARG_COMPONENT}_USER_FILELIST ${ignored} "%config(noreplace) ${INSTALL_SYSCONF2DIR}/*" PARENT_SCOPE)
ENDIF()
diff --git a/cmake/ssl.cmake b/cmake/ssl.cmake
index d407ea3d26f..65abb27923b 100644
--- a/cmake/ssl.cmake
+++ b/cmake/ssl.cmake
@@ -1,4 +1,5 @@
-# Copyright (c) 2009, 2012, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2009, 2012, Oracle and/or its affiliates.
+# Copyright (c) 2011, 2017, MariaDB Corporation
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -11,7 +12,7 @@
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
# We support different versions of SSL:
# - "bundled" uses source code in /extra/yassl
@@ -159,7 +160,7 @@ MACRO (MYSQL_CHECK_SSL)
HAVE_EncryptAes128Gcm)
ELSE()
IF(WITH_SSL STREQUAL "system")
- MESSAGE(SEND_ERROR "Cannot find appropriate system libraries for SSL. Use WITH_SSL=bundled to enable SSL support")
+ MESSAGE(SEND_ERROR "Cannot find appropriate system libraries for SSL. Use WITH_SSL=bundled to enable SSL support")
ENDIF()
MYSQL_USE_BUNDLED_SSL()
ENDIF()
diff --git a/cmake/submodules.cmake b/cmake/submodules.cmake
new file mode 100644
index 00000000000..672a3affc1d
--- /dev/null
+++ b/cmake/submodules.cmake
@@ -0,0 +1,31 @@
+# update submodules automatically
+IF(GIT_EXECUTABLE AND EXISTS "${CMAKE_SOURCE_DIR}/.git")
+ EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" config --get cmake.update-submodules
+ WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+ OUTPUT_VARIABLE cmake_update_submodules
+ RESULT_VARIABLE git_config_get_result)
+ IF(git_config_get_result EQUAL 128 OR cmake_update_submodules MATCHES no)
+ SET(update_result 0)
+ ELSEIF (cmake_update_submodules MATCHES force)
+ MESSAGE(STATUS "Updating submodules (forced)")
+ EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule update --init --force
+ WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+ RESULT_VARIABLE update_result)
+ ELSEIF (cmake_update_submodules MATCHES yes)
+ EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule update --init
+ WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+ RESULT_VARIABLE update_result)
+ ELSE()
+ MESSAGE(STATUS "Updating submodules")
+ EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule update --init
+ WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+ RESULT_VARIABLE update_result)
+ ENDIF()
+ENDIF()
+
+IF(update_result OR NOT EXISTS ${CMAKE_SOURCE_DIR}/libmariadb/CMakeLists.txt)
+ MESSAGE(FATAL_ERROR "No MariaDB Connector/C! Run
+ git submodule update --init
+Then restart the build.
+")
+ENDIF()
diff --git a/cmake/systemd.cmake b/cmake/systemd.cmake
index 841b52280e2..6986cb748ce 100644
--- a/cmake/systemd.cmake
+++ b/cmake/systemd.cmake
@@ -70,9 +70,11 @@ MACRO(CHECK_SYSTEMD)
UNSET(HAVE_SYSTEMD_SD_NOTIFYF)
MESSAGE_ONCE(systemd "Systemd features not enabled")
IF(WITH_SYSTEMD STREQUAL "yes")
- MESSAGE(FATAL_ERROR "Requested WITH_SYSTEMD=YES however no dependencies installed/found")
+ MESSAGE(FATAL_ERROR "Requested WITH_SYSTEMD=yes however no dependencies installed/found")
ENDIF()
ENDIF()
+ ELSEIF(NOT WITH_SYSTEMD STREQUAL "no")
+ MESSAGE(FATAL_ERROR "Invalid value for WITH_SYSTEMD. Must be 'yes', 'no', or 'auto'.")
ENDIF()
ENDIF()
ENDMACRO()
diff --git a/cmake/wsrep.cmake b/cmake/wsrep.cmake
index 0a1c7dd9697..e6d1379aea3 100644
--- a/cmake/wsrep.cmake
+++ b/cmake/wsrep.cmake
@@ -26,7 +26,7 @@ ENDIF()
OPTION(WITH_WSREP "WSREP replication API (to use, e.g. Galera Replication library)" ${with_wsrep_default})
# Set the patch version
-SET(WSREP_PATCH_VERSION "19")
+SET(WSREP_PATCH_VERSION "20")
# Obtain wsrep API version
FILE(STRINGS "${MySQL_SOURCE_DIR}/wsrep/wsrep_api.h" WSREP_API_VERSION
@@ -41,4 +41,3 @@ SET(WSREP_PROC_INFO ${WITH_WSREP})
IF(WITH_WSREP)
SET(WSREP_PATCH_VERSION "wsrep_${WSREP_VERSION}")
ENDIF()
-
diff --git a/config.h.cmake b/config.h.cmake
index c0fad4a3efe..51abd4b11d5 100644
--- a/config.h.cmake
+++ b/config.h.cmake
@@ -576,21 +576,4 @@
#define __STDC_FORMAT_MACROS
#endif
-/*
- stat structure (from ) is conditionally defined
- to have different layout and size depending on the defined macros.
- The correct macro is defined in my_config.h, which means it MUST be
- included first (or at least before - so, practically,
- before including any system headers).
-
- Check the include order by looking at __GLIBC__ (defined in )
-
- But we cannot force all third-party clients/connectors to include
- my_config.h first. So, their crashes are their responsibility,
- we enable this check only for MariaDB sources (SAFE_MUTEX check).
-*/
-#if defined(__GLIBC__) && defined(SAFE_MUTEX)
-#error MUST be included first!
-#endif
-
#endif
diff --git a/debian/additions/enable_encryption.preset b/debian/additions/enable_encryption.preset
new file mode 100644
index 00000000000..287bc03413a
--- /dev/null
+++ b/debian/additions/enable_encryption.preset
@@ -0,0 +1,20 @@
+#
+# !include this file into your my.cnf (or any of *.cnf files in /etc/mysql/conf.d)
+# and it will enable data at rest encryption. This is a simple way to
+# ensure that everything that can be encrypted will be and your
+# data will not leak unencrypted.
+#
+# DO NOT EDIT THIS FILE! On MariaDB upgrades it might be replaced with a
+# newer version and your edits will be lost. Instead, add your edits
+# to the .cnf file after the !include directive.
+#
+# NOTE that you also need to install an encryption plugin for the encryption
+# to work. See https://mariadb.com/kb/en/mariadb/data-at-rest-encryption/#encryption-key-management
+#
+[mariadb]
+aria-encrypt-tables
+encrypt-binlog
+encrypt-tmp-disk-tables
+encrypt-tmp-files
+loose-innodb-encrypt-log
+loose-innodb-encrypt-tables
diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh
index b6d1711e6ea..92c68c225d7 100755
--- a/debian/autobake-deb.sh
+++ b/debian/autobake-deb.sh
@@ -81,6 +81,11 @@ if [[ $GCCVERSION -lt 40800 ]] || [[ $(arch) =~ i[346]86 ]]
then
sed '/Package: mariadb-plugin-rocksdb/,+7d' -i debian/control
fi
+if [[ $GCCVERSION -lt 40800 ]]
+then
+ sed '/Package: mariadb-plugin-aws-key-management-10.2/,+13d' -i debian/control
+fi
+
# Adjust changelog, add new version
echo "Incrementing changelog and starting build scripts"
diff --git a/debian/control b/debian/control
index c501078f678..e2d99d61ad4 100644
--- a/debian/control
+++ b/debian/control
@@ -21,7 +21,7 @@ Build-Depends: bison,
libpam0g-dev,
libpcre3-dev (>= 2:8.35-3.2~),
libreadline-gplv2-dev,
- libssl-dev,
+ libssl-dev | libssl1.0-dev,
libsnappy-dev,
libsystemd-dev,
libxml2-dev,
@@ -30,6 +30,7 @@ Build-Depends: bison,
po-debconf,
psmisc,
unixodbc-dev,
+ uuid-dev,
zlib1g-dev (>= 1:1.1.3-5~)
Standards-Version: 3.8.2
Homepage: http://mariadb.org/
@@ -47,13 +48,9 @@ Conflicts: mariadb-galera-server-10.0 (<< 10.0.5),
mariadb-server-5.2,
mariadb-server-5.3,
mariadb-server-5.5 (<< 5.5.33)
-Replaces: libmariadbclient18,
- libmysqlclient18,
- libmysqlclient19,
+Replaces: libmysqlclient19,
libmysqlclient20
-Provides: libmariadbclient18,
- libmysqlclient18,
- libmysqlclient19,
+Provides: libmysqlclient19,
libmysqlclient20
Description: MariaDB database client library
MariaDB is a fast, stable and true multi-user, multi-threaded SQL database
@@ -67,27 +64,29 @@ Package: libmariadbclient18
Section: libs
Architecture: any
Depends: libmariadb3 (= ${binary:Version}), ${misc:Depends}
+Replaces: libmariadbclient18
+Provides: libmariadbclient18
Description: Virtual package to satisfy external depends
MariaDB is a fast, stable and true multi-user, multi-threaded SQL database
server. SQL (Structured Query Language) is the most popular database query
language in the world. The main goals of MariaDB are speed, robustness and
ease of use.
.
- This is an empty package that depends on the libmariadb3
- package.
+ This package provides compatibility symlinks for libmariadb3
Package: libmysqlclient18
Section: libs
Architecture: any
Depends: libmariadb3 (= ${binary:Version}), ${misc:Depends}
+Replaces: libmysqlclient18
+Provides: libmysqlclient18
Description: Virtual package to satisfy external depends
MariaDB is a fast, stable and true multi-user, multi-threaded SQL database
server. SQL (Structured Query Language) is the most popular database query
language in the world. The main goals of MariaDB are speed, robustness and
ease of use.
.
- This is an empty package that depends on the libmariadb3
- package.
+ This package provides compatibility symlinks for libmariadb3
Package: libmariadb-dev
Architecture: any
@@ -173,8 +172,7 @@ Description: MariaDB database common files (e.g. /etc/mysql/conf.d/mariadb.cnf)
Package: mariadb-client-core-10.2
Architecture: any
-Depends: libmariadb3 (>= ${source:Version}),
- mariadb-common (>= ${source:Version}),
+Depends: mariadb-common (>= ${source:Version}),
${misc:Depends},
${shlibs:Depends}
Conflicts: mariadb-client-10.0,
@@ -237,7 +235,6 @@ Description: MariaDB database core client binaries
Package: mariadb-client-10.2
Architecture: any
Depends: debianutils (>=1.6),
- libmariadb3 (>= ${source:Version}),
mariadb-client-core-10.2 (>= ${source:Version}),
mariadb-common,
${misc:Depends},
@@ -291,8 +288,7 @@ Description: MariaDB database client binaries
Package: mariadb-server-core-10.2
Architecture: any
-Depends: libmariadb3 (>= ${binary:Version}),
- mariadb-common (>= ${source:Version}),
+Depends: mariadb-common (>= ${source:Version}),
${misc:Depends},
${shlibs:Depends}
Conflicts: mariadb-server-core-10.0,
@@ -552,6 +548,16 @@ Breaks: mariadb-gssapi-client-10.1, mariadb-gssapi-client-10.2
Replaces: mariadb-gssapi-client-10.1, mariadb-gssapi-client-10.2
Description: GSSAPI authentication plugin for MariaDB client
+Package: mariadb-backup-10.2
+Section: database
+Architecture: any
+Breaks: mariadb-backup-10.1
+Replaces: mariadb-backup-10.1
+Depends: mariadb-server-10.2,
+ ${misc:Depends},
+ ${shlibs:Depends}
+Description: Backup tool for MariaDB server
+
Package: mariadb-plugin-cracklib-password-check
Architecture: any
Depends: libcrack2 (>= 2.9.0),
@@ -562,6 +568,20 @@ Description: CrackLib Password Validation Plugin for MariaDB
This password validation plugin uses cracklib to allow only
sufficiently secure (as defined by cracklib) user passwords in MariaDB.
+Package: mariadb-plugin-aws-key-management-10.2
+Section: database
+Architecture: any
+Breaks: mariadb-aws-key-management-10.1
+Replaces: mariadb-aws-key-management-10.1
+Depends: mariadb-server-10.2,
+ libcurl3,
+ ${misc:Depends},
+ ${shlibs:Depends}
+Description: Amazon Web Service Key Management Service Plugin for MariaDB
+ This encryption key management plugin gives an interface to the Amazon Web
+ Services Key Management Service for managing encryption keys used for MariaDB
+ data-at-rest encryption.
+
Package: mariadb-test
Architecture: any
Depends: mariadb-client-10.2 (= ${binary:Version}),
diff --git a/debian/libmariadb-dev.install b/debian/libmariadb-dev.install
index e11eceb704e..e62aad1b43f 100644
--- a/debian/libmariadb-dev.install
+++ b/debian/libmariadb-dev.install
@@ -1,7 +1,8 @@
usr/bin/mysql_config
-usr/include/mysql/*.h
-usr/include/mysql/psi/*.h
+usr/include/mysql
usr/lib/*/libmariadb.so
+usr/lib/*/libmysqlclient.so
+usr/lib/*/libmariadbclient.so
usr/lib/*/libmariadbclient.a
usr/lib/*/libmysqlservices.a
usr/share/aclocal/mysql.m4
diff --git a/debian/libmariadb-dev.links b/debian/libmariadb-dev.links
deleted file mode 100644
index 0076791dcfa..00000000000
--- a/debian/libmariadb-dev.links
+++ /dev/null
@@ -1,2 +0,0 @@
-usr/lib/libmysqlclient.so.18 usr/lib/libmysqlclient.so
-usr/lib/libmysqlclient_r.so.18 usr/lib/libmysqlclient_r.so
diff --git a/debian/libmariadb3.install b/debian/libmariadb3.install
index cdb1e5918df..58e2bd1dcb2 100644
--- a/debian/libmariadb3.install
+++ b/debian/libmariadb3.install
@@ -1,5 +1,5 @@
-usr/lib/*/libmariadbclient.so.*
-usr/lib/*/libmysqlclient.so.*
+usr/lib/*/libmysqlclient.so.19
+usr/lib/*/libmysqlclient.so.20
usr/lib/*/libmariadb.so.*
usr/lib/mysql/plugin/dialog.so
usr/lib/mysql/plugin/mysql_clear_password.so
diff --git a/debian/libmariadbclient18.install b/debian/libmariadbclient18.install
new file mode 100644
index 00000000000..e3939fa4fc7
--- /dev/null
+++ b/debian/libmariadbclient18.install
@@ -0,0 +1 @@
+usr/lib/*/libmariadbclient.so.18
diff --git a/debian/libmysqlclient18.install b/debian/libmysqlclient18.install
new file mode 100644
index 00000000000..3193873093f
--- /dev/null
+++ b/debian/libmysqlclient18.install
@@ -0,0 +1 @@
+usr/lib/*/libmysqlclient.so.18
diff --git a/debian/mariadb-backup-10.2.install b/debian/mariadb-backup-10.2.install
new file mode 100644
index 00000000000..734117c92e2
--- /dev/null
+++ b/debian/mariadb-backup-10.2.install
@@ -0,0 +1,2 @@
+usr/bin/mariabackup
+usr/bin/mbstream
diff --git a/debian/mariadb-plugin-aws-key-management-10.2.install b/debian/mariadb-plugin-aws-key-management-10.2.install
new file mode 100644
index 00000000000..ed966b4115d
--- /dev/null
+++ b/debian/mariadb-plugin-aws-key-management-10.2.install
@@ -0,0 +1,2 @@
+usr/lib/mysql/plugin/aws_key_management.so
+debian/additions/enable_encryption.preset etc/mysql/conf.d/
diff --git a/debian/mariadb-server-10.2.install b/debian/mariadb-server-10.2.install
index 3b5a475f486..81981fe1f60 100644
--- a/debian/mariadb-server-10.2.install
+++ b/debian/mariadb-server-10.2.install
@@ -38,6 +38,7 @@ usr/bin/wsrep_sst_mysqldump
usr/bin/wsrep_sst_rsync
usr/bin/wsrep_sst_xtrabackup
usr/bin/wsrep_sst_xtrabackup-v2
+usr/bin/wsrep_sst_mariabackup
usr/lib/mysql/plugin/auth_pam.so
usr/lib/mysql/plugin/auth_socket.so
usr/lib/mysql/plugin/file_key_management.so
diff --git a/debian/mariadb-server-core-10.2.install b/debian/mariadb-server-core-10.2.install
index 9ab77b2f95a..d882bd53f52 100644
--- a/debian/mariadb-server-core-10.2.install
+++ b/debian/mariadb-server-core-10.2.install
@@ -10,6 +10,7 @@ usr/share/mysql/estonian
usr/share/mysql/french
usr/share/mysql/german
usr/share/mysql/greek
+usr/share/mysql/hindi
usr/share/mysql/hungarian
usr/share/mysql/italian
usr/share/mysql/japanese
diff --git a/debian/patches/50_mysql-test__db_test.dpatch b/debian/patches/50_mysql-test__db_test.dpatch
index 670afbf14ac..14c0fc4acf6 100755
--- a/debian/patches/50_mysql-test__db_test.dpatch
+++ b/debian/patches/50_mysql-test__db_test.dpatch
@@ -11,14 +11,14 @@
--- old/mysql-test/mysql-test-run.pl 2009-06-16 14:24:09.000000000 +0200
+++ new/mysql-test/mysql-test-run.pl 2009-07-04 00:03:34.000000000 +0200
@@ -3578,6 +3578,11 @@ sub mysql_install_db {
- mtr_appendfile_to_file("$sql_dir/mysql_system_tables_data.sql",
- $bootstrap_sql_file);
+ mtr_appendfile_to_file("$sql_dir/mysql_system_tables_data.sql",
+ $bootstrap_sql_file);
-+ mtr_tofile($bootstrap_sql_file, "-- Debian removed the default privileges on the 'test' database\n");
-+ mtr_tofile($bootstrap_sql_file, "INSERT INTO mysql.db VALUES ('%','test','','Y','Y','Y','Y','Y','Y','N','Y','Y','Y','Y','Y','Y','Y','Y','N','N','Y','Y');\n");
-+ mtr_tofile($bootstrap_sql_file, "INSERT INTO mysql.db VALUES ('%','test\\_%','','Y','Y','Y','Y','Y','Y','N','Y','Y','Y','Y','Y','Y','Y','Y','N','N','Y','Y');\n");
-+
++ mtr_tofile($bootstrap_sql_file, "-- Debian removed the default privileges on the 'test' database\n");
++ mtr_tofile($bootstrap_sql_file, "INSERT INTO mysql.db VALUES ('%','test','','Y','Y','Y','Y','Y','Y','N','Y','Y','Y','Y','Y','Y','Y','Y','N','N','Y','Y');\n");
++ mtr_tofile($bootstrap_sql_file, "INSERT INTO mysql.db VALUES ('%','test\\_%','','Y','Y','Y','Y','Y','Y','N','Y','Y','Y','Y','Y','Y','Y','Y','N','N','Y','Y');\n");
+
- # Add test data for timezone - this is just a subset, on a real
- # system these tables will be populated either by mysql_tzinfo_to_sql
- # or by downloading the timezone table package from our website
++
+ # Add test data for timezone - this is just a subset, on a real
+ # system these tables will be populated either by mysql_tzinfo_to_sql
+ # or by downloading the timezone table package from our website
diff --git a/debian/rules b/debian/rules
index 7aa09ea988b..2f385b26455 100755
--- a/debian/rules
+++ b/debian/rules
@@ -162,6 +162,7 @@ override_dh_auto_install:
install -D -m 644 debian/mariadb-server-10.2.py $(TMP)/usr/share/apport/package-hooks/source_mariadb-10.2.py
# Install libmariadbclient18 compatibility links
+ ln -s /usr/lib/$(DEB_HOST_MULTIARCH)/libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmariadbclient.so
ln -s /usr/lib/$(DEB_HOST_MULTIARCH)/libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmariadbclient.so.18
# Install libmysqlclientclientXX compatibility links
diff --git a/extra/CMakeLists.txt b/extra/CMakeLists.txt
index 9cc5e7faa86..8c73e23d72b 100644
--- a/extra/CMakeLists.txt
+++ b/extra/CMakeLists.txt
@@ -79,7 +79,6 @@ IF(WITH_INNOBASE_STORAGE_ENGINE OR WITH_XTRADB_STORAGE_ENGINE)
../storage/innobase/ut/ut0ut.cc
../storage/innobase/buf/buf0buf.cc
../storage/innobase/page/page0zip.cc
- ../storage/innobase/os/os0file.cc
../storage/innobase/fil/fil0crypt.cc
)
diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc
index 238a5a80ac5..a0976970db7 100644
--- a/extra/innochecksum.cc
+++ b/extra/innochecksum.cc
@@ -26,40 +26,58 @@
Published with a permission.
*/
-#include
#include
#include
#include
#include
#include
#include
-#ifdef HAVE_UNISTD_H
+#ifndef __WIN__
# include
#endif
#include
#include
-#include /* ORACLE_WELCOME_COPYRIGHT_NOTICE */
+#include /* ORACLE_WELCOME_COPYRIGHT_NOTICE */
/* Only parts of these files are included from the InnoDB codebase.
The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */
-#include "univ.i" /* include all of this */
-#include "page0size.h" /* page_size_t */
-#include "page0zip.h" /* page_zip_calc_checksum() */
-#include "page0page.h" /* PAGE_* */
-#include "trx0undo.h" /* TRX_UNDO_* */
-#include "fut0lst.h" /* FLST_NODE_SIZE */
-#include "buf0checksum.h" /* buf_calc_page_*() */
-#include "fil0fil.h" /* FIL_* */
-#include "fil0crypt.h"
-#include "os0file.h"
-#include "fsp0fsp.h" /* fsp_flags_get_page_size() &
- fsp_flags_get_zip_size() */
-#include "mach0data.h" /* mach_read_from_4() */
-#include "ut0crc32.h" /* ut_crc32_init() */
-#include "fsp0pagecompress.h" /* fil_get_compression_alg_name */
+typedef void fil_space_t;
+
+#include "univ.i" /* include all of this */
+#include "page0size.h"
+
+#define FLST_BASE_NODE_SIZE (4 + 2 * FIL_ADDR_SIZE)
+#define FLST_NODE_SIZE (2 * FIL_ADDR_SIZE)
+#define FSEG_PAGE_DATA FIL_PAGE_DATA
+#define FSEG_HEADER_SIZE 10
+#define UT_BITS_IN_BYTES(b) (((b) + 7) / 8)
+
+#include "ut0ut.h"
#include "ut0byte.h"
+#include "mtr0types.h"
#include "mach0data.h"
+#include "fsp0types.h"
+#include "rem0rec.h"
+#include "buf0checksum.h" /* buf_calc_page_*() */
+#include "buf0buf.h" /* buf_page_is_corrupted */
+#include "fil0fil.h" /* FIL_* */
+#include "page0page.h" /* PAGE_* */
+#include "page0zip.h" /* page_zip_*() */
+#include "trx0undo.h" /* TRX_* */
+#include "fsp0fsp.h" /* fsp_flags_get_page_size() &
+ fsp_flags_get_zip_size() */
+#include "ut0crc32.h" /* ut_crc32_init() */
+#include "fsp0pagecompress.h" /* fil_get_compression_alg_name */
+#include "fil0crypt.h" /* fil_space_verify_crypt_checksum */
+
+#include
+
+#ifdef UNIV_NONINL
+# include "fsp0fsp.ic"
+# include "mach0data.ic"
+# include "ut0rnd.ic"
+#endif
#ifndef PRIuMAX
#define PRIuMAX "llu"
@@ -68,17 +86,21 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */
/* Global variables */
static bool verbose;
static bool just_count;
-static uintmax_t start_page;
-static uintmax_t end_page;
-static uintmax_t do_page;
+static unsigned long long start_page;
+static unsigned long long end_page;
+static unsigned long long do_page;
static bool use_end_page;
static bool do_one_page;
-/* replaces declaration in srv0srv.c */
-ulong srv_page_size;
-page_size_t univ_page_size(0, 0, false);
+static my_bool do_leaf;
+static my_bool per_page_details;
+static ulong n_merge;
extern ulong srv_checksum_algorithm;
+static ulong physical_page_size; /* Page size in bytes on disk. */
+static ulong logical_page_size; /* Page size when uncompressed. */
+ulong srv_page_size;
+page_size_t univ_page_size(0, 0, false);
/* Current page number (0 based). */
-uintmax_t cur_page_num;
+unsigned long long cur_page_num;
/* Skip the checksum verification. */
static bool no_check;
/* Enabled for strict checksum verification. */
@@ -86,7 +108,7 @@ bool strict_verify = 0;
/* Enabled for rewrite checksum. */
static bool do_write;
/* Mismatches count allowed (0 by default). */
-static uintmax_t allow_mismatches;
+static unsigned long long allow_mismatches=0;
static bool page_type_summary;
static bool page_type_dump;
/* Store filename for page-type-dump option. */
@@ -99,8 +121,7 @@ char* log_filename = NULL;
FILE* log_file = NULL;
/* Enabled for log write option. */
static bool is_log_enabled = false;
-static my_bool do_leaf;
-static ulong n_merge;
+
#ifndef _WIN32
/* advisory lock for non-window system. */
struct flock lk;
@@ -198,75 +219,6 @@ struct per_index_stats {
std::map index_ids;
-bool encrypted = false;
-
-ulint
-page_is_comp(
-/*=========*/
- const page_t* page) /*!< in: index page */
-{
- return(page_header_get_field(page, PAGE_N_HEAP) & 0x8000);
-}
-
-bool
-page_is_leaf(
-/*=========*/
- const page_t* page) /*!< in: page */
-{
- return(!*(const uint16*) (page + (PAGE_HEADER + PAGE_LEVEL)));
-}
-
-ulint
-page_get_page_no(
-/*=============*/
- const page_t* page) /*!< in: page */
-{
- return(mach_read_from_4(page + FIL_PAGE_OFFSET));
-}
-#define FSEG_HEADER_SIZE 10 /*!< Length of the file system
- header, in bytes */
-#define REC_N_NEW_EXTRA_BYTES 5
-#define REC_N_OLD_EXTRA_BYTES 6
-#define PAGE_DATA (PAGE_HEADER + 36 + 2 * FSEG_HEADER_SIZE)
- /* start of data on the page */
-#define PAGE_NEW_SUPREMUM (PAGE_DATA + 2 * REC_N_NEW_EXTRA_BYTES + 8)
- /* offset of the page supremum record on a
- new-style compact page */
-#define PAGE_NEW_SUPREMUM_END (PAGE_NEW_SUPREMUM + 8)
-#define PAGE_OLD_SUPREMUM (PAGE_DATA + 2 + 2 * REC_N_OLD_EXTRA_BYTES + 8)
- /* offset of the page supremum record on an
- old-style page */
-#define PAGE_OLD_SUPREMUM_END (PAGE_OLD_SUPREMUM + 9)
- /* offset of the page supremum record end on
- an old-style page */
-#define FLST_BASE_NODE_SIZE (4 + 2 * 6)
-#define XDES_ARR_OFFSET (FSP_HEADER_OFFSET + FSP_HEADER_SIZE)
-#define XDES_FREE_BIT 0
-#define XDES_BITMAP (FLST_NODE_SIZE + 12)
-#define XDES_BITS_PER_PAGE 2
-#define UT_BITS_IN_BYTES(b) (((b) + 7) / 8)
-#define XDES_SIZE \
- (XDES_BITMAP \
- + UT_BITS_IN_BYTES(FSP_EXTENT_SIZE * XDES_BITS_PER_PAGE))
-
-ulint
-page_get_data_size(
-/*===============*/
- const page_t* page) /*!< in: index page */
-{
- ulint ret;
-
- ret = (ulint)(page_header_get_field(page, PAGE_HEAP_TOP)
- - (page_is_comp(page)
- ? PAGE_NEW_SUPREMUM_END
- : PAGE_OLD_SUPREMUM_END)
- - page_header_get_field(page, PAGE_GARBAGE));
-
- ut_ad(ret < UNIV_PAGE_SIZE);
-
- return(ret);
-}
-
void print_index_leaf_stats(
unsigned long long id,
const per_index_stats& index,
@@ -280,7 +232,7 @@ void print_index_leaf_stats(
fprintf(fil_out, "page_no\tdata_size\tn_recs\n");
while (it_page != index.leaves.end()) {
const per_page_stats& stat = it_page->second;
- fprintf(fil_out, "%llu\t%lu\t%lu\n", it_page->first, stat.data_size, stat.n_recs);
+ fprintf(fil_out, "%llu\t" ULINTPF "\t" ULINTPF "\n", it_page->first, stat.data_size, stat.n_recs);
page_no = stat.right_page_no;
it_page = index.leaves.find(page_no);
}
@@ -306,6 +258,7 @@ void defrag_analysis(
break;
}
}
+
if (index.max_data_size) {
n_pages += data_size_total / index.max_data_size;
if (data_size_total % index.max_data_size != 0) {
@@ -315,10 +268,14 @@ void defrag_analysis(
}
if (index.leaf_pages) {
- fprintf(fil_out, "count = %lu free = %lu\n", index.count, index.free_pages);
+ fprintf(fil_out, "count = " ULINTPF " free = " ULINTPF "\n", index.count, index.free_pages);
}
- fprintf(fil_out, "%llu\t\t%llu\t\t%lu\t\t%lu\t\t%lu\t\t%.2f\t%lu\n",
+ if (!n_leaf_pages) {
+ n_leaf_pages = 1;
+ }
+
+ fprintf(fil_out, "%llu\t\t%llu\t\t" ULINTPF "\t\t" ULINTPF "\t\t" ULINTPF "\t\t%.2f\t" ULINTPF "\n",
id, index.leaf_pages, n_leaf_pages, n_merge, n_pages,
1.0 - (double)n_pages / (double)n_leaf_pages, index.max_data_size);
}
@@ -368,29 +325,6 @@ get_page_size(
return(page_size_t(flags));
}
-#ifdef MYSQL_COMPRESSION
-/** Decompress a page
-@param[in,out] buf Page read from disk, uncompressed data will
- also be copied to this page
-@param[in, out] scratch Page to use for temporary decompress
-@param[in] page_size scratch physical size
-@return true if decompress succeeded */
-static
-bool page_decompress(
- byte* buf,
- byte* scratch,
- page_size_t page_size)
-{
- dberr_t err;
-
- /* Set the dblwr recover flag to false. */
- err = os_file_decompress_page(
- false, buf, scratch, page_size.physical());
-
- return(err == DB_SUCCESS);
-}
-#endif
-
#ifdef _WIN32
/***********************************************//*
@param [in] error error no. from the getLastError().
@@ -435,6 +369,7 @@ open_file(
access = GENERIC_READ;
flags = _O_RDONLY | _O_BINARY;
}
+
/* CreateFile() also provide advisory lock with the usage of
access and share mode of the file.*/
hFile = CreateFile(
@@ -458,8 +393,7 @@ open_file(
if (do_write) {
create_flag = O_RDWR;
lk.l_type = F_WRLCK;
- }
- else {
+ } else {
create_flag = O_RDONLY;
lk.l_type = F_RDLCK;
}
@@ -521,12 +455,18 @@ ulong read_file(
/** Check if page is corrupted or not.
@param[in] buf page frame
@param[in] page_size page size
+@param[in] is_encrypted true if page0 contained cryp_data
+ with crypt_scheme encrypted
+@param[in] is_compressed true if page0 fsp_flags contained
+ page compression flag
@retval true if page is corrupted otherwise false. */
static
bool
is_page_corrupted(
- const byte* buf,
- const page_size_t& page_size)
+ byte* buf,
+ const page_size_t& page_size,
+ bool is_encrypted,
+ bool is_compressed)
{
/* enable if page is corrupted. */
@@ -534,8 +474,22 @@ is_page_corrupted(
/* use to store LSN values. */
ulint logseq;
ulint logseqfield;
+ ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE);
+ uint key_version = mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
+ ulint space_id = mach_read_from_4(
+ buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
- if (!page_size.is_compressed()) {
+ /* We can't trust only a page type, thus we take account
+ also fsp_flags or crypt_data on page 0 */
+ if ((page_type == FIL_PAGE_PAGE_COMPRESSED && is_compressed) ||
+ (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED &&
+ is_compressed && is_encrypted)) {
+ /* Page compressed tables do not contain post compression
+ checksum. */
+ return (false);
+ }
+
+ if (page_size.is_compressed()) {
/* check the stored log sequence numbers
for uncompressed tablespace. */
logseq = mach_read_from_4(buf + FIL_PAGE_LSN + 4);
@@ -545,35 +499,32 @@ is_page_corrupted(
if (is_log_enabled) {
fprintf(log_file,
- "page::%" PRIuMAX
+ "space::" ULINTPF " page::%llu"
"; log sequence number:first = " ULINTPF
"; second = " ULINTPF "\n",
- cur_page_num, logseq, logseqfield);
+ space_id, cur_page_num, logseq, logseqfield);
if (logseq != logseqfield) {
fprintf(log_file,
- "Fail; page %" PRIuMAX
+ "Fail; space::" ULINTPF " page::%llu"
" invalid (fails log "
"sequence number check)\n",
- cur_page_num);
+ space_id, cur_page_num);
}
}
}
- /* FIXME: Read the page number from the tablespace header,
- and check that every page carries the same page number. */
+ /* Again we can't trust only FIL_PAGE_FILE_FLUSH_LSN field
+ now repurposed as FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION,
+ we need to check also crypt_data contents.
- /* If page is encrypted, use different checksum calculation
+ If page is encrypted, use different checksum calculation
as innochecksum can't decrypt pages. Note that some old InnoDB
versions did not initialize FIL_PAGE_FILE_FLUSH_LSN field
so if crypt checksum does not match we verify checksum using
- normal method.
- */
- if (mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0) {
- is_corrupted = fil_space_verify_crypt_checksum(
- const_cast(buf), page_size,
- mach_read_from_4(buf
- + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
- cur_page_num);
+ normal method. */
+ if (is_encrypted && key_version != 0) {
+ is_corrupted = !fil_space_verify_crypt_checksum(buf,
+ page_size, space_id, cur_page_num);
} else {
is_corrupted = true;
}
@@ -624,8 +575,9 @@ is_page_empty(
if (*page++) {
return (false);
}
- }
- return (true);
+ }
+
+ return (true);
}
/********************************************************************//**
@@ -678,7 +630,7 @@ update_checksum(
mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum);
if (is_log_enabled) {
- fprintf(log_file, "page::%" PRIuMAX "; Updated checksum ="
+ fprintf(log_file, "page::%llu; Updated checksum ="
" %u\n", cur_page_num, checksum);
}
@@ -709,7 +661,7 @@ update_checksum(
mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum);
if (is_log_enabled) {
- fprintf(log_file, "page::%" PRIuMAX "; Updated checksum field1"
+ fprintf(log_file, "page::%llu; Updated checksum field1"
" = %u\n", cur_page_num, checksum);
}
@@ -723,7 +675,7 @@ update_checksum(
FIL_PAGE_END_LSN_OLD_CHKSUM,checksum);
if (is_log_enabled) {
- fprintf(log_file, "page::%" PRIuMAX "; Updated checksum "
+ fprintf(log_file, "page::%llu; Updated checksum "
"field2 = %u\n", cur_page_num, checksum);
}
@@ -798,7 +750,7 @@ write_file(
if (page_size
!= fwrite(buf, 1, page_size, file == stdin ? stdout : file)) {
- fprintf(stderr, "Failed to write page %" PRIuMAX " to %s: %s\n",
+ fprintf(stderr, "Failed to write page::%llu to %s: %s\n",
cur_page_num, filename, strerror(errno));
return(false);
@@ -815,93 +767,32 @@ write_file(
return(true);
}
-/********************************************************//**
-Gets the next index page number.
-@return next page number */
-ulint
-btr_page_get_next(
-/*==============*/
- const page_t* page) /*!< in: index page */
-{
- return(mach_read_from_4(page + FIL_PAGE_NEXT));
-}
-
-/********************************************************//**
-Gets the previous index page number.
-@return prev page number */
-ulint
-btr_page_get_prev(
-/*==============*/
- const page_t* page) /*!< in: index page */
-{
- return(mach_read_from_4(page + FIL_PAGE_PREV));
-}
-
-ulint
-mach_read_ulint(
- const byte* ptr,
- mlog_id_t type)
-{
- switch (type) {
- case MLOG_1BYTE:
- return(mach_read_from_1(ptr));
- case MLOG_2BYTES:
- return(mach_read_from_2(ptr));
- case MLOG_4BYTES:
- return(mach_read_from_4(ptr));
- default:
- break;
- }
-
- ut_error;
- return(0);
-}
-ibool
-xdes_get_bit(
-/*=========*/
- const xdes_t* descr, /*!< in: descriptor */
- ulint bit, /*!< in: XDES_FREE_BIT or XDES_CLEAN_BIT */
- ulint offset) /*!< in: page offset within extent:
- 0 ... FSP_EXTENT_SIZE - 1 */
-{
- ulint index = bit + XDES_BITS_PER_PAGE * offset;
-
- ulint bit_index = index % 8;
- ulint byte_index = index / 8;
-
- return(ut_bit_get_nth(
- mach_read_ulint(descr + XDES_BITMAP + byte_index,
- MLOG_1BYTE),
- bit_index));
-}
-
/*
Parse the page and collect/dump the information about page type
@param [in] page buffer page
-@param [in] xdes xdes page
+@param [out] xdes extend descriptor page
@param [in] file file for diagnosis.
-@param [in] page_size page size
+@param [in] page_size page_size
+@param [in] is_encrypted tablespace is encrypted
*/
void
parse_page(
const byte* page,
- const byte* xdes,
+ byte* xdes,
FILE* file,
- page_size_t page_size)
+ const page_size_t& page_size,
+ bool is_encrypted)
{
- unsigned long long id=0;
- ulint undo_page_type=0;
- ulint n_recs;
- ulint page_no=0;
- ulint right_page_no=0;
- ulint left_page_no=0;
- ulint data_bytes=0;
- bool is_leaf=false;
- ulint size_range_id=0;
- ulint data_types=0;
- ulint key_version = 0;
-
+ unsigned long long id;
+ ulint undo_page_type;
char str[20]={'\0'};
+ ulint n_recs;
+ ulint page_no;
+ ulint left_page_no;
+ ulint right_page_no;
+ ulint data_bytes;
+ bool is_leaf;
+ int size_range_id;
/* Check whether page is doublewrite buffer. */
if(skip_page) {
@@ -912,102 +803,105 @@ parse_page(
switch (mach_read_from_2(page + FIL_PAGE_TYPE)) {
- case FIL_PAGE_INDEX:
+ case FIL_PAGE_INDEX: {
+ uint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
page_type.n_fil_page_index++;
- id = mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID);
- n_recs = page_header_get_field(page, PAGE_N_RECS);
- page_no = page_get_page_no(page);
- left_page_no = btr_page_get_prev(page);
- right_page_no = btr_page_get_next(page);
- key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
/* If page is encrypted we can't read index header */
- if (!key_version) {
- data_bytes = page_get_data_size(page);
+ if (!is_encrypted) {
+ id = mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID);
+ n_recs = mach_read_from_2(page + PAGE_HEADER + PAGE_N_RECS);
+ page_no = mach_read_from_4(page + FIL_PAGE_OFFSET);
+ left_page_no = mach_read_from_4(page + FIL_PAGE_PREV);
+ right_page_no = mach_read_from_4(page + FIL_PAGE_NEXT);
+ ulint is_comp = mach_read_from_2(page + PAGE_HEADER + PAGE_N_HEAP) & 0x8000;
+ ulint level = mach_read_from_2(page + PAGE_HEADER + PAGE_LEVEL);
+ ulint garbage = mach_read_from_2(page + PAGE_HEADER + PAGE_GARBAGE);
+
+
+ data_bytes = (ulint)(mach_read_from_2(page + PAGE_HEADER + PAGE_HEAP_TOP)
+ - (is_comp
+ ? PAGE_NEW_SUPREMUM_END
+ : PAGE_OLD_SUPREMUM_END)
+ - garbage);
+
+ is_leaf = (!*(const uint16*) (page + (PAGE_HEADER + PAGE_LEVEL)));
+
+ if (page_type_dump) {
+ fprintf(file, "#::%llu\t\t|\t\tIndex page\t\t\t|"
+ "\tindex id=%llu,", cur_page_num, id);
+
+ fprintf(file,
+ " page level=" ULINTPF
+ ", No. of records=" ULINTPF
+ ", garbage=" ULINTPF ", %s\n",
+ level, n_recs, garbage, str);
+ }
+
+ size_range_id = (data_bytes * SIZE_RANGES_FOR_PAGE
+ + page_size.logical() - 1) /
+ page_size.logical();
+
+ if (size_range_id > SIZE_RANGES_FOR_PAGE + 1) {
+ /* data_bytes is bigger than logical_page_size */
+ size_range_id = SIZE_RANGES_FOR_PAGE + 1;
+ }
+ if (per_page_details) {
+ printf("index id=%llu page " ULINTPF " leaf %d n_recs " ULINTPF " data_bytes " ULINTPF
+ "\n", id, page_no, is_leaf, n_recs, data_bytes);
+ }
+ /* update per-index statistics */
+ {
+ if (index_ids.count(id) == 0) {
+ index_ids[id] = per_index_stats();
+ }
+ std::map::iterator it;
+ it = index_ids.find(id);
+ per_index_stats &index = (it->second);
+ const byte* des = xdes + XDES_ARR_OFFSET
+ + XDES_SIZE * ((page_no & (page_size.physical() - 1))
+ / FSP_EXTENT_SIZE);
+ if (xdes_get_bit(des, XDES_FREE_BIT,
+ page_no % FSP_EXTENT_SIZE)) {
+ index.free_pages++;
+ return;
+ }
+
+ index.pages++;
+
+ if (is_leaf) {
+ index.leaf_pages++;
+ if (data_bytes > index.max_data_size) {
+ index.max_data_size = data_bytes;
+ }
+ struct per_page_stats pp(n_recs, data_bytes,
+ left_page_no, right_page_no);
+
+ index.leaves[page_no] = pp;
+
+ if (left_page_no == ULINT32_UNDEFINED) {
+ index.first_leaf_page = page_no;
+ index.count++;
+ }
+ }
+
+ index.total_n_recs += n_recs;
+ index.total_data_bytes += data_bytes;
+ index.pages_in_size_range[size_range_id] ++;
+ }
} else {
- data_bytes = 0;
- }
-
- is_leaf = page_is_leaf(page);
-
- if (page_type_dump) {
- fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tIndex page\t\t\t|"
- "\tindex id=%llu,", cur_page_num, id);
-
- fprintf(file,
- " page level=" ULINTPF " leaf %u"
- ", No. of records=" ULINTPF
- ", garbage=" ULINTPF
- ", n_recs=" ULINTPF
- ", %s\n",
- page_header_get_field(page, PAGE_LEVEL),
- is_leaf,
- n_recs,
- page_header_get_field(page, PAGE_GARBAGE),
- data_types,
- str);
- }
-
- size_range_id = (data_bytes * SIZE_RANGES_FOR_PAGE
- + page_size.logical() - 1) /
- page_size.logical();
-
- if (size_range_id > SIZE_RANGES_FOR_PAGE + 1) {
- /* data_bytes is bigger than logical_page_size */
- size_range_id = SIZE_RANGES_FOR_PAGE + 1;
- }
-
- /* update per-index statistics */
- {
- if (index_ids.count(id) == 0) {
- index_ids[id] = per_index_stats();
- }
-
- std::map::iterator it;
- it = index_ids.find(id);
- per_index_stats &index = (it->second);
- uchar* des = (uchar *)(xdes + XDES_ARR_OFFSET
- + XDES_SIZE * ((page_no & (page_size.physical() - 1))
- / FSP_EXTENT_SIZE));
-
- if (xdes_get_bit(des, XDES_FREE_BIT,
- page_no % FSP_EXTENT_SIZE)) {
- index.free_pages++;
- return;
- }
-
- index.pages++;
-
- if (is_leaf) {
- index.leaf_pages++;
- if (data_bytes > index.max_data_size) {
- index.max_data_size = data_bytes;
- }
-
- struct per_page_stats pp(n_recs, data_bytes,
- left_page_no, right_page_no);
-
- index.leaves[page_no] = pp;
-
- if (left_page_no == ULINT32_UNDEFINED) {
- index.first_leaf_page = page_no;
- index.count++;
- }
- }
-
- index.total_n_recs += n_recs;
- index.total_data_bytes += data_bytes;
- index.pages_in_size_range[size_range_id] ++;
+ fprintf(file, "#::%llu\t\t|\t\tEncrypted Index page\t\t\t|"
+ "\tkey_version %u,%s\n", cur_page_num, key_version, str);
}
break;
-
+ }
case FIL_PAGE_UNDO_LOG:
page_type.n_fil_page_undo_log++;
undo_page_type = mach_read_from_2(page +
TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_TYPE);
if (page_type_dump) {
- fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tUndo log page\t\t\t|",
+ fprintf(file, "#::%llu\t\t|\t\tUndo log page\t\t\t|",
cur_page_num);
}
if (undo_page_type == TRX_UNDO_INSERT) {
@@ -1081,7 +975,7 @@ parse_page(
case FIL_PAGE_INODE:
page_type.n_fil_page_inode++;
if (page_type_dump) {
- fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInode page\t\t\t|"
+ fprintf(file, "#::%llu\t\t|\t\tInode page\t\t\t|"
"\t%s\n",cur_page_num, str);
}
break;
@@ -1089,7 +983,7 @@ parse_page(
case FIL_PAGE_IBUF_FREE_LIST:
page_type.n_fil_page_ibuf_free_list++;
if (page_type_dump) {
- fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInsert buffer free list"
+ fprintf(file, "#::%llu\t\t|\t\tInsert buffer free list"
" page\t|\t%s\n", cur_page_num, str);
}
break;
@@ -1097,7 +991,7 @@ parse_page(
case FIL_PAGE_TYPE_ALLOCATED:
page_type.n_fil_page_type_allocated++;
if (page_type_dump) {
- fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tFreshly allocated "
+ fprintf(file, "#::%llu\t\t|\t\tFreshly allocated "
"page\t\t|\t%s\n", cur_page_num, str);
}
break;
@@ -1105,7 +999,7 @@ parse_page(
case FIL_PAGE_IBUF_BITMAP:
page_type.n_fil_page_ibuf_bitmap++;
if (page_type_dump) {
- fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInsert Buffer "
+ fprintf(file, "#::%llu\t\t|\t\tInsert Buffer "
"Bitmap\t\t|\t%s\n", cur_page_num, str);
}
break;
@@ -1113,7 +1007,7 @@ parse_page(
case FIL_PAGE_TYPE_SYS:
page_type.n_fil_page_type_sys++;
if (page_type_dump) {
- fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tSystem page\t\t\t|"
+ fprintf(file, "#::%llu\t\t|\t\tSystem page\t\t\t|"
"\t%s\n",cur_page_num, str);
}
break;
@@ -1121,25 +1015,25 @@ parse_page(
case FIL_PAGE_TYPE_TRX_SYS:
page_type.n_fil_page_type_trx_sys++;
if (page_type_dump) {
- fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tTransaction system "
+ fprintf(file, "#::%llu\t\t|\t\tTransaction system "
"page\t\t|\t%s\n", cur_page_num, str);
}
break;
case FIL_PAGE_TYPE_FSP_HDR:
page_type.n_fil_page_type_fsp_hdr++;
- memcpy((void *)xdes, (void *)page, page_size.physical());
+ memcpy(xdes, page, page_size.physical());
if (page_type_dump) {
- fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tFile Space "
+ fprintf(file, "#::%llu\t\t|\t\tFile Space "
"Header\t\t|\t%s\n", cur_page_num, str);
}
break;
case FIL_PAGE_TYPE_XDES:
page_type.n_fil_page_type_xdes++;
- memcpy((void *)xdes, (void *)page, page_size.physical());
+ memcpy(xdes, page, page_size.physical());
if (page_type_dump) {
- fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tExtent descriptor "
+ fprintf(file, "#::%llu\t\t|\t\tExtent descriptor "
"page\t\t|\t%s\n", cur_page_num, str);
}
break;
@@ -1147,7 +1041,7 @@ parse_page(
case FIL_PAGE_TYPE_BLOB:
page_type.n_fil_page_type_blob++;
if (page_type_dump) {
- fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tBLOB page\t\t\t|\t%s\n",
+ fprintf(file, "#::%llu\t\t|\t\tBLOB page\t\t\t|\t%s\n",
cur_page_num, str);
}
break;
@@ -1155,7 +1049,7 @@ parse_page(
case FIL_PAGE_TYPE_ZBLOB:
page_type.n_fil_page_type_zblob++;
if (page_type_dump) {
- fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tCompressed BLOB "
+ fprintf(file, "#::%llu\t\t|\t\tCompressed BLOB "
"page\t\t|\t%s\n", cur_page_num, str);
}
break;
@@ -1163,25 +1057,26 @@ parse_page(
case FIL_PAGE_TYPE_ZBLOB2:
page_type.n_fil_page_type_zblob2++;
if (page_type_dump) {
- fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tSubsequent Compressed "
+ fprintf(file, "#::%llu\t\t|\t\tSubsequent Compressed "
"BLOB page\t|\t%s\n", cur_page_num, str);
}
break;
- case FIL_PAGE_PAGE_COMPRESSED:
- page_type.n_fil_page_type_page_compressed++;
- if (page_type_dump) {
- fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tPage compressed "
+ case FIL_PAGE_PAGE_COMPRESSED:
+ page_type.n_fil_page_type_page_compressed++;
+ if (page_type_dump) {
+ fprintf(file, "#::%llu\t\t|\t\tPage compressed "
"page\t|\t%s\n", cur_page_num, str);
- }
- break;
- case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED:
- page_type.n_fil_page_type_page_compressed_encrypted++;
- if (page_type_dump) {
- fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tPage compressed encrypted "
+ }
+ break;
+
+ case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED:
+ page_type.n_fil_page_type_page_compressed_encrypted++;
+ if (page_type_dump) {
+ fprintf(file, "#::%llu\t\t|\t\tPage compressed encrypted "
"page\t|\t%s\n", cur_page_num, str);
- }
- break;
+ }
+ break;
default:
page_type.n_fil_page_type_other++;
break;
@@ -1269,7 +1164,7 @@ print_summary(
page_type.n_fil_page_type_page_compressed);
fprintf(fil_out, "%8d\tPage compressed encrypted page\n",
page_type.n_fil_page_type_page_compressed_encrypted);
- fprintf(fil_out, "%8d\tOther type of page",
+ fprintf(fil_out, "%8d\tOther type of page\n",
page_type.n_fil_page_type_other);
fprintf(fil_out, "\n===============================================\n");
@@ -1327,8 +1222,10 @@ static struct my_option innochecksum_options[] = {
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{"verbose", 'v', "Verbose (prints progress every 5 seconds).",
&verbose, &verbose, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+#ifndef DBUG_OFF
{"debug", '#', "Output debug log. See " REFMAN "dbug-package.html",
&dbug_setting, &dbug_setting, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
+#endif /* !DBUG_OFF */
{"count", 'c', "Print the count of pages in the file and exits.",
&just_count, &just_count, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"start_page", 's', "Start on this page number (0 based).",
@@ -1357,10 +1254,12 @@ static struct my_option innochecksum_options[] = {
{"page-type-dump", 'D', "Dump the page type info for each page in a "
"tablespace.", &page_dump_filename, &page_dump_filename, 0,
GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+ {"per-page-details", 'i', "Print out per-page detail information.",
+ &per_page_details, &per_page_details, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"log", 'l', "log output.",
&log_filename, &log_filename, 0,
GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
- {"leaf", 'e', "Examine leaf index pages",
+ {"leaf", 'f', "Examine leaf index pages",
&do_leaf, &do_leaf, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"merge", 'm', "leaf page count if merge given number of consecutive pages",
&n_merge, &n_merge, 0, GET_ULONG, REQUIRED_ARG, 0, 0, (longlong)10L, 0, 1, 0},
@@ -1388,9 +1287,9 @@ static void usage(void)
puts(ORACLE_WELCOME_COPYRIGHT_NOTICE("2000"));
printf("InnoDB offline file checksum utility.\n");
printf("Usage: %s [-c] [-s ] [-e ] "
- "[-p ] [-v] [-a ] [-n] "
+ "[-p ] [-i] [-v] [-a ] [-n] "
"[-C ] [-w ] [-S] [-D ] "
- "[-l ] [-e] \n", my_progname);
+ "[-l ] [-l] [-m ] \n", my_progname);
printf("See " REFMAN "innochecksum.html for usage hints.\n");
my_print_help(innochecksum_options);
my_print_variables(innochecksum_options);
@@ -1483,7 +1382,7 @@ get_options(
char ***argv)
{
if (handle_options(argc, argv, innochecksum_options,
- innochecksum_get_one_option)) {
+ innochecksum_get_one_option)) {
my_end(0);
exit(true);
}
@@ -1498,6 +1397,133 @@ get_options(
return (false);
}
+/** Check from page 0 if table is encrypted.
+@param[in] filename Filename
+@param[in] page_size page size
+@param[in] page Page 0
+@retval true if tablespace is encrypted, false if not
+*/
+static
+bool check_encryption(
+ const char* filename,
+ const page_size_t& page_size,
+ byte * page)
+{
+ ulint offset = (FSP_HEADER_OFFSET + (XDES_ARR_OFFSET + XDES_SIZE *
+ (page_size.physical()) / FSP_EXTENT_SIZE));
+
+ if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) {
+ return false;
+ }
+
+ ulint type = mach_read_from_1(page + offset + MAGIC_SZ + 0);
+
+ if (! (type == CRYPT_SCHEME_UNENCRYPTED ||
+ type == CRYPT_SCHEME_1)) {
+ return false;
+ }
+
+ ulint iv_length = mach_read_from_1(page + offset + MAGIC_SZ + 1);
+
+ if (iv_length != CRYPT_SCHEME_1_IV_LEN) {
+ return false;
+ }
+
+ uint min_key_version = mach_read_from_4
+ (page + offset + MAGIC_SZ + 2 + iv_length);
+
+ uint key_id = mach_read_from_4
+ (page + offset + MAGIC_SZ + 2 + iv_length + 4);
+
+ if (type == CRYPT_SCHEME_1 && is_log_enabled) {
+ fprintf(log_file,"Tablespace %s encrypted key_version %u key_id %u\n",
+ filename, min_key_version, key_id);
+ }
+
+ return (type == CRYPT_SCHEME_1);
+}
+
+/**
+Verify page checksum.
+@param[in] buf page to verify
+@param[in] page_size page size
+@param[in] is_encrypted true if tablespace is encrypted
+@param[in] is_compressed true if tablespace is page compressed
+@param[in,out] mismatch_count Number of pages failed in checksum verify
+@retval 0 if page checksum matches or 1 if it does not match
+*/
+static
+int verify_checksum(
+ byte* buf,
+ const page_size_t& page_size,
+ bool is_encrypted,
+ bool is_compressed,
+ unsigned long long* mismatch_count)
+{
+ int exit_status = 0;
+ bool is_corrupted = false;
+
+ is_corrupted = is_page_corrupted(
+ buf, page_size, is_encrypted, is_compressed);
+
+ if (is_corrupted) {
+ fprintf(stderr, "Fail: page::%llu invalid\n",
+ cur_page_num);
+
+ (*mismatch_count)++;
+
+ if (*mismatch_count > allow_mismatches) {
+ fprintf(stderr,
+ "Exceeded the "
+ "maximum allowed "
+ "checksum mismatch "
+ "count::%llu current::%llu\n",
+ *mismatch_count,
+ allow_mismatches);
+
+ exit_status = 1;
+ }
+ }
+
+ return (exit_status);
+}
+
+/** Rewrite page checksum if needed.
+@param[in] filename File name
+@param[in] fil_in File pointer
+@param[in] buf page
+@param[in] page_size page size
+@param[in] pos File position
+@param[in] is_encrypted true if tablespace is encrypted
+@param[in] is_compressed true if tablespace is page compressed
+@retval 0 if checksum rewrite was successful, 1 if error was detected */
+static
+int
+rewrite_checksum(
+ const char* filename,
+ FILE* fil_in,
+ byte* buf,
+ const page_size_t& page_size,
+ fpos_t* pos,
+ bool is_encrypted,
+ bool is_compressed)
+{
+ int exit_status = 0;
+ /* Rewrite checksum. Note that for encrypted and
+ page compressed tables this is not currently supported. */
+ if (do_write &&
+ !is_encrypted &&
+ !is_compressed
+ && !write_file(filename, fil_in, buf,
+ page_size.is_compressed(), pos,
+ static_cast(page_size.physical()))) {
+
+ exit_status = 1;
+ }
+
+ return (exit_status);
+}
+
int main(
int argc,
char **argv)
@@ -1507,15 +1533,12 @@ int main(
/* our input filename. */
char* filename;
/* Buffer to store pages read. */
+ byte* buf_ptr = NULL;
+ byte* xdes_ptr = NULL;
byte* buf = NULL;
- /* Buffer for xdes */
byte* xdes = NULL;
/* bytes read count */
ulong bytes;
- /* Buffer to decompress page.*/
-#ifdef MYSQL_COMPRESSION
- byte* tbuf = NULL;
-#endif
/* current time */
time_t now;
/* last time */
@@ -1527,6 +1550,8 @@ int main(
struct stat st;
#endif /* _WIN32 */
+ int exit_status = 0;
+
/* size of file (has to be 64 bits) */
unsigned long long int size = 0;
/* number of pages in file */
@@ -1534,9 +1559,7 @@ int main(
off_t offset = 0;
/* count the no. of page corrupted. */
- ulint mismatch_count = 0;
- /* Variable to ack the page is corrupted or not. */
- bool is_corrupted = false;
+ unsigned long long mismatch_count = 0;
bool partial_page_read = false;
/* Enabled when read from stdin is done. */
@@ -1556,32 +1579,37 @@ int main(
DBUG_PROCESS(argv[0]);
if (get_options(&argc,&argv)) {
- DBUG_RETURN(1);
+ exit_status = 1;
+ goto my_exit;
}
if (strict_verify && no_check) {
fprintf(stderr, "Error: --strict-check option cannot be used "
"together with --no-check option.\n");
- DBUG_RETURN(1);
+ exit_status = 1;
+ goto my_exit;
}
if (no_check && !do_write) {
fprintf(stderr, "Error: --no-check must be associated with "
"--write option.\n");
- DBUG_RETURN(1);
+ exit_status = 1;
+ goto my_exit;
}
if (page_type_dump) {
fil_page_type = create_file(page_dump_filename);
if (!fil_page_type) {
- DBUG_RETURN(1);
+ exit_status = 1;
+ goto my_exit;
}
}
if (is_log_enabled) {
log_file = create_file(log_filename);
if (!log_file) {
- DBUG_RETURN(1);
+ exit_status = 1;
+ goto my_exit;
}
fprintf(log_file, "InnoDB File Checksum Utility.\n");
}
@@ -1591,17 +1619,17 @@ int main(
}
- buf = (byte*) malloc(UNIV_PAGE_SIZE_MAX * 2);
- xdes = (byte *) malloc(UNIV_PAGE_SIZE_MAX *2);
-#ifdef MYSQL_COMPRESSION
- tbuf = buf + UNIV_PAGE_SIZE_MAX;
-#endif
+ buf_ptr = (byte*) malloc(UNIV_PAGE_SIZE_MAX * 2);
+ xdes_ptr = (byte*)malloc(UNIV_PAGE_SIZE_MAX * 2);
+ buf = (byte *) ut_align(buf_ptr, UNIV_PAGE_SIZE_MAX);
+ xdes = (byte *) ut_align(xdes_ptr, UNIV_PAGE_SIZE_MAX);
+
/* The file name is not optional. */
for (int i = 0; i < argc; ++i) {
+
/* Reset parameters for each file. */
filename = argv[i];
memset(&page_type, 0, sizeof(innodb_page_type));
- is_corrupted = false;
partial_page_read = false;
skip_page = false;
@@ -1626,7 +1654,8 @@ int main(
fprintf(stderr, "Error: %s cannot be found\n",
filename);
- goto err_exit;
+ exit_status = 1;
+ goto my_exit;
}
if (!read_from_stdin) {
@@ -1634,35 +1663,17 @@ int main(
fil_in = open_file(filename);
/*If fil_in is NULL, terminate as some error encountered */
if(fil_in == NULL) {
- goto err_exit;
+ exit_status = 1;
+ goto my_exit;
}
/* Save the current file pointer in pos variable.*/
if (0 != fgetpos(fil_in, &pos)) {
perror("fgetpos");
- goto err_exit;
+ exit_status = 1;
+ goto my_exit;
}
}
- /* Testing for lock mechanism. The innochecksum
- acquire lock on given file. So other tools accessing the same
- file for processsing must fail. */
-#ifdef _WIN32
- DBUG_EXECUTE_IF("innochecksum_cause_mysqld_crash",
- ut_ad(page_dump_filename);
- while((_access( page_dump_filename, 0)) == 0) {
- sleep(1);
- }
- DBUG_RETURN(0); );
-#else
- DBUG_EXECUTE_IF("innochecksum_cause_mysqld_crash",
- ut_ad(page_dump_filename);
- struct stat status_buf;
- while(stat(page_dump_filename, &status_buf) == 0) {
- sleep(1);
- }
- DBUG_RETURN(0); );
-#endif /* _WIN32 */
-
/* Read the minimum page size. */
bytes = ulong(fread(buf, 1, UNIV_ZIP_SIZE_MIN, fil_in));
partial_page_read = true;
@@ -1670,10 +1681,11 @@ int main(
if (bytes != UNIV_ZIP_SIZE_MIN) {
fprintf(stderr, "Error: Was not able to read the "
"minimum page size ");
- fprintf(stderr, "of %d bytes. Bytes read was %lu\n",
+ fprintf(stderr, "of %d bytes. Bytes read was " ULINTPF "\n",
UNIV_ZIP_SIZE_MIN, bytes);
- goto err_exit;
+ exit_status = 1;
+ goto my_exit;
}
/* enable variable is_system_tablespace when space_id of given
@@ -1683,8 +1695,79 @@ int main(
FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, 4))
? true : false;
+ /* Determine page size, zip_size and page compression
+ from fsp_flags and encryption metadata from page 0 */
const page_size_t& page_size = get_page_size(buf);
+ ulint flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf);
+ ulint zip_size = page_size.is_compressed() ? page_size.logical() : 0;
+ logical_page_size = page_size.is_compressed() ? zip_size : 0;
+ physical_page_size = page_size.physical();
+ srv_page_size = page_size.logical();
+ bool is_compressed = FSP_FLAGS_HAS_PAGE_COMPRESSION(flags);
+
+ if (page_size.physical() > UNIV_ZIP_SIZE_MIN) {
+ /* Read rest of the page 0 to determine crypt_data */
+ bytes = ulong(read_file(buf, partial_page_read, page_size.physical(), fil_in));
+
+ if (bytes != page_size.physical()) {
+ fprintf(stderr, "Error: Was not able to read the "
+ "rest of the page ");
+ fprintf(stderr, "of " ULINTPF " bytes. Bytes read was " ULINTPF "\n",
+ page_size.physical() - UNIV_ZIP_SIZE_MIN, bytes);
+
+ exit_status = 1;
+ goto my_exit;
+ }
+ partial_page_read = false;
+ }
+
+ /* Now that we have full page 0 in buffer, check encryption */
+ bool is_encrypted = check_encryption(filename, page_size, buf);
+
+ /* Verify page 0 contents. Note that we can't allow
+ checksum mismatch on page 0, because that would mean we
+ could not trust it content. */
+ if (!no_check) {
+ unsigned long long tmp_allow_mismatches = allow_mismatches;
+ allow_mismatches = 0;
+
+ exit_status = verify_checksum(buf, page_size, is_encrypted, is_compressed, &mismatch_count);
+
+ if (exit_status) {
+ fprintf(stderr, "Error: Page 0 checksum mismatch, can't continue. \n");
+ goto my_exit;
+ }
+ allow_mismatches = tmp_allow_mismatches;
+ }
+
+ if ((exit_status = rewrite_checksum(filename, fil_in, buf,
+ page_size, &pos, is_encrypted, is_compressed))) {
+ goto my_exit;
+ }
+
+ if (page_type_dump) {
+ fprintf(fil_page_type,
+ "\n\nFilename::%s\n", filename);
+ fprintf(fil_page_type,
+ "========================================"
+ "======================================\n");
+ fprintf(fil_page_type,
+ "\tPAGE_NO\t\t|\t\tPAGE_TYPE\t\t"
+ "\t|\tEXTRA INFO\n");
+ fprintf(fil_page_type,
+ "========================================"
+ "======================================\n");
+ }
+
+ if (per_page_details) {
+ printf("page %llu ", cur_page_num);
+ }
+
+ if (page_type_summary || page_type_dump) {
+ parse_page(buf, xdes, fil_page_type, page_size, is_encrypted);
+ }
+
pages = (ulint) (size / page_size.physical());
if (just_count) {
@@ -1700,14 +1783,14 @@ int main(
"(" ULINTPF " pages)\n", filename, size, pages);
if (do_one_page) {
fprintf(log_file, "Innochecksum: "
- "checking page %" PRIuMAX "\n",
+ "checking page::%llu;\n",
do_page);
}
}
} else {
if (is_log_enabled) {
fprintf(log_file, "Innochecksum: checking "
- "pages in range %" PRIuMAX " to %" PRIuMAX "\n",
+ "pages in range::%llu to %llu\n",
start_page, use_end_page ?
end_page : (pages - 1));
}
@@ -1731,14 +1814,16 @@ int main(
perror("Error: Unable to seek to "
"necessary offset");
- goto err_exit;
+ exit_status = 1;
+ goto my_exit;
}
/* Save the current file pointer in
pos variable. */
if (0 != fgetpos(fil_in, &pos)) {
perror("fgetpos");
- goto err_exit;
+ exit_status = 1;
+ goto my_exit;
}
} else {
@@ -1758,7 +1843,7 @@ int main(
bytes = read_file(buf,
partial_page_read,
static_cast(
- page_size.physical()),
+ page_size.physical()),
fil_in);
partial_page_read = false;
@@ -1769,34 +1854,22 @@ int main(
"to seek to necessary "
"offset");
- goto err_exit;
+ exit_status = 1;
+ goto my_exit;
}
}
}
}
- if (page_type_dump) {
- fprintf(fil_page_type,
- "\n\nFilename::%s\n", filename);
- fprintf(fil_page_type,
- "========================================"
- "======================================\n");
- fprintf(fil_page_type,
- "\tPAGE_NO\t\t|\t\tPAGE_TYPE\t\t"
- "\t|\tEXTRA INFO\n");
- fprintf(fil_page_type,
- "========================================"
- "======================================\n");
- }
-
/* main checksumming loop */
- cur_page_num = start_page;
+ cur_page_num = start_page ? start_page : cur_page_num + 1;
+
lastt = 0;
while (!feof(fil_in)) {
bytes = read_file(buf, partial_page_read,
static_cast(
- page_size.physical()), fil_in);
+ page_size.physical()), fil_in);
partial_page_read = false;
if (!bytes && feof(fil_in)) {
@@ -1808,14 +1881,16 @@ int main(
page_size.physical());
perror(" ");
- goto err_exit;
+ exit_status = 1;
+ goto my_exit;
}
if (bytes != page_size.physical()) {
- fprintf(stderr, "Error: bytes read (%lu) "
+ fprintf(stderr, "Error: bytes read (" ULINTPF ") "
"doesn't match page size (" ULINTPF ")\n",
bytes, page_size.physical());
- goto err_exit;
+ exit_status = 1;
+ goto my_exit;
}
if (is_system_tablespace) {
@@ -1823,60 +1898,29 @@ int main(
skip_page = is_page_doublewritebuffer(buf);
} else {
skip_page = false;
-#ifdef MYSQL_COMPRESSION
- if (!page_decompress(buf, tbuf, page_size)) {
-
- fprintf(stderr,
- "Page decompress failed");
-
- goto err_exit;
- }
-#endif
}
- ulint page_type = mach_read_from_2(buf + FIL_PAGE_TYPE);
+ ulint cur_page_type = mach_read_from_2(buf+FIL_PAGE_TYPE);
- if (page_type == FIL_PAGE_PAGE_COMPRESSED ||
- page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
+ /* FIXME: Page compressed or Page compressed and encrypted
+ pages do not contain checksum. */
+ if (cur_page_type == FIL_PAGE_PAGE_COMPRESSED ||
+ cur_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
skip_page = true;
}
/* If no-check is enabled, skip the
checksum verification.*/
- if (!no_check) {
- /* Checksum verification */
- if (!skip_page) {
- is_corrupted = is_page_corrupted(
- buf, page_size);
-
- if (is_corrupted) {
- fprintf(stderr, "Fail: page "
- "%" PRIuMAX " invalid\n",
- cur_page_num);
-
- mismatch_count++;
-
- if(mismatch_count > allow_mismatches) {
- fprintf(stderr,
- "Exceeded the "
- "maximum allowed "
- "checksum mismatch "
- "count::%" PRIuMAX "\n",
- allow_mismatches);
-
- goto err_exit;
- }
- }
- }
+ if (!no_check
+ && !skip_page
+ && (exit_status = verify_checksum(buf, page_size,
+ is_encrypted, is_compressed, &mismatch_count))) {
+ goto my_exit;
}
- /* Rewrite checksum */
- if (do_write
- && !write_file(filename, fil_in, buf,
- page_size.is_compressed(), &pos,
- static_cast(page_size.physical()))) {
-
- goto err_exit;
+ if ((exit_status = rewrite_checksum(filename, fil_in, buf,
+ page_size, &pos, is_encrypted, is_compressed))) {
+ goto my_exit;
}
/* end if this was the last page we were supposed to check */
@@ -1884,12 +1928,17 @@ int main(
break;
}
+ if (per_page_details) {
+ printf("page %llu ", cur_page_num);
+ }
+
if (page_type_summary || page_type_dump) {
- parse_page(buf, xdes , fil_page_type, page_size);
+ parse_page(buf, xdes, fil_page_type, page_size, is_encrypted);
}
/* do counter increase and progress printing */
cur_page_num++;
+
if (verbose && !read_from_stdin) {
if ((cur_page_num % 64) == 0) {
now = time(0);
@@ -1898,7 +1947,7 @@ int main(
}
if (now - lastt >= 1
&& is_log_enabled) {
- fprintf(log_file, "page %" PRIuMAX " "
+ fprintf(log_file, "page::%llu "
"okay: %.3f%% done\n",
(cur_page_num - 1),
(float) cur_page_num / pages * 100);
@@ -1929,19 +1978,19 @@ int main(
fclose(log_file);
}
- free(buf);
- free(xdes);
- my_end(0);
+ free(buf_ptr);
+ free(xdes_ptr);
- DBUG_RETURN(0);
+ my_end(exit_status);
+ DBUG_RETURN(exit_status);
-err_exit:
- if (buf) {
- free(buf);
+my_exit:
+ if (buf_ptr) {
+ free(buf_ptr);
}
- if (xdes) {
- free(xdes);
+ if (xdes_ptr) {
+ free(xdes_ptr);
}
if (!read_from_stdin && fil_in) {
@@ -1952,7 +2001,6 @@ err_exit:
fclose(log_file);
}
- my_end(1);
-
- DBUG_RETURN(1);
+ my_end(exit_status);
+ DBUG_RETURN(exit_status);
}
diff --git a/extra/mariabackup/CMakeLists.txt b/extra/mariabackup/CMakeLists.txt
new file mode 100644
index 00000000000..78188d2f8e8
--- /dev/null
+++ b/extra/mariabackup/CMakeLists.txt
@@ -0,0 +1,117 @@
+# Copyright (c) 2013, 2017 Percona LLC and/or its affiliates.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+
+OPTION(WITH_MARIABACKUP "Include mariabackup" ON)
+IF(NOT WITH_MARIABACKUP)
+ RETURN()
+ENDIF()
+
+
+IF(NOT WIN32)
+ CHECK_SYMBOL_EXISTS(regcomp regex.h HAVE_SYSTEM_REGEX)
+ IF(HAVE_SYSTEM_REGEX)
+ ADD_DEFINITIONS(-DHAVE_SYSTEM_REGEX)
+ ENDIF()
+ENDIF()
+
+
+INCLUDE_DIRECTORIES(
+ ${CMAKE_SOURCE_DIR}/include
+ ${CMAKE_SOURCE_DIR}/sql
+ ${CMAKE_CURRENT_SOURCE_DIR}/quicklz
+ ${CMAKE_CURRENT_SOURCE_DIR}
+ ${CMAKE_CURRENT_SOURCE_DIR}/crc
+ )
+
+IF(NOT HAVE_SYSTEM_REGEX)
+ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/pcre)
+ENDIF()
+
+ADD_DEFINITIONS(-UMYSQL_SERVER)
+########################################################################
+# xtrabackup binary
+########################################################################
+
+IF(WIN32)
+ SET(NT_SERVICE_SOURCE ${PROJECT_SOURCE_DIR}/sql/nt_servc.cc)
+ELSE()
+ SET(NT_SERVICE_SOURCE)
+ENDIF()
+
+ADD_DEFINITIONS(-DPCRE_STATIC=1)
+
+MYSQL_ADD_EXECUTABLE(mariabackup
+ xtrabackup.cc
+ innobackupex.cc
+ changed_page_bitmap.cc
+ datasink.c
+ ds_buffer.c
+ ds_compress.c
+ ds_local.c
+ ds_stdout.c
+ ds_tmpfile.c
+ ds_xbstream.c
+ fil_cur.cc
+ quicklz/quicklz.c
+ read_filt.cc
+ write_filt.cc
+ wsrep.cc
+ xbstream_write.c
+ backup_mysql.cc
+ backup_copy.cc
+ encryption_plugin.cc
+ ${PROJECT_SOURCE_DIR}/sql/net_serv.cc
+ ${NT_SERVICE_SOURCE}
+ ${PROJECT_SOURCE_DIR}/libmysqld/libmysql.c
+ COMPONENT backup
+ )
+
+
+# Export all symbols on Unix, for better crash callstacks
+SET_TARGET_PROPERTIES(mariabackup PROPERTIES ENABLE_EXPORTS TRUE)
+ADD_SUBDIRECTORY(crc)
+
+
+TARGET_LINK_LIBRARIES(mariabackup sql crc)
+
+IF(NOT HAVE_SYSTEM_REGEX)
+ TARGET_LINK_LIBRARIES(mariabackup pcreposix)
+ENDIF()
+
+
+########################################################################
+# xbstream binary
+########################################################################
+MYSQL_ADD_EXECUTABLE(mbstream
+ ds_buffer.c
+ ds_local.c
+ ds_stdout.c
+ datasink.c
+ xbstream.c
+ xbstream_read.c
+ xbstream_write.c
+ COMPONENT backup
+ )
+
+
+TARGET_LINK_LIBRARIES(mbstream
+ mysys
+ crc
+)
+
+IF(MSVC)
+ SET_TARGET_PROPERTIES(mbstream PROPERTIES LINK_FLAGS setargv.obj)
+ENDIF()
diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc
new file mode 100644
index 00000000000..19cb768cd01
--- /dev/null
+++ b/extra/mariabackup/backup_copy.cc
@@ -0,0 +1,2017 @@
+/******************************************************
+hot backup tool for InnoDB
+(c) 2009-2015 Percona LLC and/or its affiliates
+(c) 2017 MariaDB
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************
+
+This file incorporates work covered by the following copyright and
+permission notice:
+
+Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA
+
+*******************************************************/
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include "fil_cur.h"
+#include "xtrabackup.h"
+#include "common.h"
+#include "backup_copy.h"
+#include "backup_mysql.h"
+#include
+#include "xb0xb.h"
+
+
+/* list of files to sync for --rsync mode */
+static std::set rsync_list;
+/* locations of tablespaces read from .isl files */
+static std::map tablespace_locations;
+
+/* Whether LOCK BINLOG FOR BACKUP has been issued during backup */
+bool binlog_locked;
+
+/************************************************************************
+Struct represents file or directory. */
+struct datadir_node_t {
+ ulint dbpath_len;
+ char *filepath;
+ ulint filepath_len;
+ char *filepath_rel;
+ ulint filepath_rel_len;
+ bool is_empty_dir;
+ bool is_file;
+};
+
+/************************************************************************
+Holds the state needed to enumerate files in MySQL data directory. */
+struct datadir_iter_t {
+ char *datadir_path;
+ char *dbpath;
+ ulint dbpath_len;
+ char *filepath;
+ ulint filepath_len;
+ char *filepath_rel;
+ ulint filepath_rel_len;
+ pthread_mutex_t mutex;
+ os_file_dir_t dir;
+ os_file_dir_t dbdir;
+ os_file_stat_t dbinfo;
+ os_file_stat_t fileinfo;
+ dberr_t err;
+ bool is_empty_dir;
+ bool is_file;
+ bool skip_first_level;
+};
+
+
+/************************************************************************
+Represents the context of the thread processing MySQL data directory. */
+struct datadir_thread_ctxt_t {
+ datadir_iter_t *it;
+ uint n_thread;
+ uint *count;
+ pthread_mutex_t count_mutex;
+ os_thread_id_t id;
+ bool ret;
+};
+
+static bool backup_files_from_datadir(const char *dir_path);
+
+/************************************************************************
+Retirn true if character if file separator */
+bool
+is_path_separator(char c)
+{
+ return(c == FN_LIBCHAR || c == FN_LIBCHAR2);
+}
+
+
+/************************************************************************
+Fill the node struct. Memory for node need to be allocated and freed by
+the caller. It is caller responsibility to initialize node with
+datadir_node_init and cleanup the memory with datadir_node_free.
+Node can not be shared between threads. */
+static
+void
+datadir_node_fill(datadir_node_t *node, datadir_iter_t *it)
+{
+ if (node->filepath_len < it->filepath_len) {
+ free(node->filepath);
+ node->filepath = (char*)(malloc(it->filepath_len));
+ node->filepath_len = it->filepath_len;
+ }
+ if (node->filepath_rel_len < it->filepath_rel_len) {
+ free(node->filepath_rel);
+ node->filepath_rel = (char*)(malloc(it->filepath_rel_len));
+ node->filepath_rel_len = it->filepath_rel_len;
+ }
+
+ strcpy(node->filepath, it->filepath);
+ strcpy(node->filepath_rel, it->filepath_rel);
+ node->is_empty_dir = it->is_empty_dir;
+ node->is_file = it->is_file;
+}
+
+static
+void
+datadir_node_free(datadir_node_t *node)
+{
+ free(node->filepath);
+ free(node->filepath_rel);
+ memset(node, 0, sizeof(datadir_node_t));
+}
+
+static
+void
+datadir_node_init(datadir_node_t *node)
+{
+ memset(node, 0, sizeof(datadir_node_t));
+}
+
+
+/************************************************************************
+Create the MySQL data directory iterator. Memory needs to be released
+with datadir_iter_free. Position should be advanced with
+datadir_iter_next_file. Iterator can be shared between multiple
+threads. It is guaranteed that each thread receives unique file from
+data directory into its local node struct. */
+static
+datadir_iter_t *
+datadir_iter_new(const char *path, bool skip_first_level = true)
+{
+ datadir_iter_t *it;
+
+ it = static_cast(malloc(sizeof(datadir_iter_t)));
+ memset(it, 0, sizeof(datadir_iter_t));
+
+ pthread_mutex_init(&it->mutex, NULL);
+ it->datadir_path = strdup(path);
+
+ it->dir = os_file_opendir(it->datadir_path, TRUE);
+
+ if (it->dir == NULL) {
+
+ goto error;
+ }
+
+ it->err = DB_SUCCESS;
+
+ it->dbpath_len = FN_REFLEN;
+ it->dbpath = static_cast(malloc(it->dbpath_len));
+
+ it->filepath_len = FN_REFLEN;
+ it->filepath = static_cast(malloc(it->filepath_len));
+
+ it->filepath_rel_len = FN_REFLEN;
+ it->filepath_rel = static_cast(malloc(it->filepath_rel_len));
+
+ it->skip_first_level = skip_first_level;
+
+ return(it);
+
+error:
+ free(it);
+
+ return(NULL);
+}
+
+static
+bool
+datadir_iter_next_database(datadir_iter_t *it)
+{
+ if (it->dbdir != NULL) {
+ if (os_file_closedir(it->dbdir) != 0) {
+
+ msg("Warning: could not"
+ " close database directory %s\n", it->dbpath);
+
+ it->err = DB_ERROR;
+
+ }
+ it->dbdir = NULL;
+ }
+
+ while (os_file_readdir_next_file(it->datadir_path,
+ it->dir, &it->dbinfo) == 0) {
+ ulint len;
+
+ if ((it->dbinfo.type == OS_FILE_TYPE_FILE
+ && it->skip_first_level)
+ || it->dbinfo.type == OS_FILE_TYPE_UNKNOWN) {
+
+ continue;
+ }
+
+ /* We found a symlink or a directory; try opening it to see
+ if a symlink is a directory */
+
+ len = strlen(it->datadir_path)
+ + strlen (it->dbinfo.name) + 2;
+ if (len > it->dbpath_len) {
+ it->dbpath_len = len;
+ free(it->dbpath);
+
+ it->dbpath = static_cast(
+ malloc(it->dbpath_len));
+ }
+ ut_snprintf(it->dbpath, it->dbpath_len,
+ "%s/%s", it->datadir_path,
+ it->dbinfo.name);
+ os_normalize_path(it->dbpath);
+
+ if (it->dbinfo.type == OS_FILE_TYPE_FILE) {
+ it->is_file = true;
+ return(true);
+ }
+
+ if (check_if_skip_database_by_path(it->dbpath)) {
+ msg("Skipping db: %s\n", it->dbpath);
+ continue;
+ }
+
+ /* We want wrong directory permissions to be a fatal error for
+ XtraBackup. */
+ it->dbdir = os_file_opendir(it->dbpath, TRUE);
+
+ if (it->dbdir != NULL) {
+
+ it->is_file = false;
+ return(true);
+ }
+
+ }
+
+ return(false);
+}
+
+/************************************************************************
+Concatenate n parts into single path */
+static
+void
+make_path_n(int n, char **path, ulint *path_len, ...)
+{
+ ulint len_needed = n + 1;
+ char *p;
+ int i;
+ va_list vl;
+
+ ut_ad(n > 0);
+
+ va_start(vl, path_len);
+ for (i = 0; i < n; i++) {
+ p = va_arg(vl, char*);
+ len_needed += strlen(p);
+ }
+ va_end(vl);
+
+ if (len_needed < *path_len) {
+ free(*path);
+ *path = static_cast(malloc(len_needed));
+ }
+
+ va_start(vl, path_len);
+ p = va_arg(vl, char*);
+ strcpy(*path, p);
+ for (i = 1; i < n; i++) {
+ size_t plen;
+ p = va_arg(vl, char*);
+ plen = strlen(*path);
+ if (!is_path_separator((*path)[plen - 1])) {
+ (*path)[plen] = FN_LIBCHAR;
+ (*path)[plen + 1] = 0;
+ }
+ strcat(*path + plen, p);
+ }
+ va_end(vl);
+}
+
+static
+bool
+datadir_iter_next_file(datadir_iter_t *it)
+{
+ if (it->is_file && it->dbpath) {
+ make_path_n(2, &it->filepath, &it->filepath_len,
+ it->datadir_path, it->dbinfo.name);
+
+ make_path_n(1, &it->filepath_rel, &it->filepath_rel_len,
+ it->dbinfo.name);
+
+ it->is_empty_dir = false;
+ it->is_file = false;
+
+ return(true);
+ }
+
+ if (!it->dbpath || !it->dbdir) {
+
+ return(false);
+ }
+
+ while (os_file_readdir_next_file(it->dbpath, it->dbdir,
+ &it->fileinfo) == 0) {
+
+ if (it->fileinfo.type == OS_FILE_TYPE_DIR) {
+
+ continue;
+ }
+
+ /* We found a symlink or a file */
+ make_path_n(3, &it->filepath, &it->filepath_len,
+ it->datadir_path, it->dbinfo.name,
+ it->fileinfo.name);
+
+ make_path_n(2, &it->filepath_rel, &it->filepath_rel_len,
+ it->dbinfo.name, it->fileinfo.name);
+
+ it->is_empty_dir = false;
+
+ return(true);
+ }
+
+ return(false);
+}
+
+static
+bool
+datadir_iter_next(datadir_iter_t *it, datadir_node_t *node)
+{
+ bool ret = true;
+
+ pthread_mutex_lock(&it->mutex);
+
+ if (datadir_iter_next_file(it)) {
+
+ datadir_node_fill(node, it);
+
+ goto done;
+ }
+
+ while (datadir_iter_next_database(it)) {
+
+ if (datadir_iter_next_file(it)) {
+
+ datadir_node_fill(node, it);
+
+ goto done;
+ }
+
+ make_path_n(2, &it->filepath, &it->filepath_len,
+ it->datadir_path, it->dbinfo.name);
+
+ make_path_n(1, &it->filepath_rel, &it->filepath_rel_len,
+ it->dbinfo.name);
+
+ it->is_empty_dir = true;
+
+ datadir_node_fill(node, it);
+
+ goto done;
+ }
+
+ /* nothing found */
+ ret = false;
+
+done:
+ pthread_mutex_unlock(&it->mutex);
+
+ return(ret);
+}
+
+/************************************************************************
+Interface to read MySQL data file sequentially. One should open file
+with datafile_open to get cursor and close the cursor with
+datafile_close. Cursor can not be shared between multiple
+threads. */
+static
+void
+datadir_iter_free(datadir_iter_t *it)
+{
+ pthread_mutex_destroy(&it->mutex);
+
+ if (it->dbdir) {
+
+ os_file_closedir(it->dbdir);
+ }
+
+ if (it->dir) {
+
+ os_file_closedir(it->dir);
+ }
+
+ free(it->dbpath);
+ free(it->filepath);
+ free(it->filepath_rel);
+ free(it->datadir_path);
+ free(it);
+}
+
+
+/************************************************************************
+Holds the state needed to copy single data file. */
+struct datafile_cur_t {
+ pfs_os_file_t file;
+ char rel_path[FN_REFLEN];
+ char abs_path[FN_REFLEN];
+ MY_STAT statinfo;
+ uint thread_n;
+ byte* orig_buf;
+ byte* buf;
+ size_t buf_size;
+ size_t buf_read;
+ size_t buf_offset;
+};
+
+static
+void
+datafile_close(datafile_cur_t *cursor)
+{
+ if (cursor->file != OS_FILE_CLOSED) {
+ os_file_close(cursor->file);
+ }
+ free(cursor->buf);
+}
+
+static
+bool
+datafile_open(const char *file, datafile_cur_t *cursor, uint thread_n)
+{
+ bool success;
+
+ memset(cursor, 0, sizeof(datafile_cur_t));
+
+ strncpy(cursor->abs_path, file, sizeof(cursor->abs_path));
+
+ /* Get the relative path for the destination tablespace name, i.e. the
+ one that can be appended to the backup root directory. Non-system
+ tablespaces may have absolute paths for remote tablespaces in MySQL
+ 5.6+. We want to make "local" copies for the backup. */
+ strncpy(cursor->rel_path,
+ xb_get_relative_path(cursor->abs_path, FALSE),
+ sizeof(cursor->rel_path));
+
+ cursor->file = os_file_create_simple_no_error_handling(
+ 0, cursor->abs_path,
+ OS_FILE_OPEN, OS_FILE_READ_ALLOW_DELETE, true, &success);
+ if (!success) {
+ /* The following call prints an error message */
+ os_file_get_last_error(TRUE);
+
+ msg("[%02u] error: cannot open "
+ "file %s\n",
+ thread_n, cursor->abs_path);
+
+ return(false);
+ }
+
+ if (!my_stat(cursor->abs_path, &cursor->statinfo, 0)) {
+ msg("[%02u] error: cannot stat %s\n",
+ thread_n, cursor->abs_path);
+
+ datafile_close(cursor);
+
+ return(false);
+ }
+
+ posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL);
+
+ cursor->buf_size = 10 * 1024 * 1024;
+ cursor->buf = static_cast(malloc((ulint)cursor->buf_size));
+
+ return(true);
+}
+
+
+static
+xb_fil_cur_result_t
+datafile_read(datafile_cur_t *cursor)
+{
+ ulint to_read;
+
+ xtrabackup_io_throttling();
+
+ to_read = (ulint)MY_MIN(cursor->statinfo.st_size - cursor->buf_offset,
+ cursor->buf_size);
+
+ if (to_read == 0) {
+ return(XB_FIL_CUR_EOF);
+ }
+
+ if (!os_file_read(IORequestRead,
+ cursor->file, cursor->buf, cursor->buf_offset,
+ to_read)) {
+ return(XB_FIL_CUR_ERROR);
+ }
+
+ posix_fadvise(cursor->file, cursor->buf_offset, to_read,
+ POSIX_FADV_DONTNEED);
+
+ cursor->buf_read = to_read;
+ cursor->buf_offset += to_read;
+
+ return(XB_FIL_CUR_SUCCESS);
+}
+
+
+
+/************************************************************************
+Check to see if a file exists.
+Takes name of the file to check.
+@return true if file exists. */
+static
+bool
+file_exists(const char *filename)
+{
+ MY_STAT stat_arg;
+
+ if (!my_stat(filename, &stat_arg, MYF(0))) {
+
+ return(false);
+ }
+
+ return(true);
+}
+
+/************************************************************************
+Trim leading slashes from absolute path so it becomes relative */
+static
+const char *
+trim_dotslash(const char *path)
+{
+ while (*path) {
+ if (is_path_separator(*path)) {
+ ++path;
+ continue;
+ }
+ if (*path == '.' && is_path_separator(path[1])) {
+ path += 2;
+ continue;
+ }
+ break;
+ }
+
+ return(path);
+}
+
+
+
+/************************************************************************
+Check if string ends with given suffix.
+@return true if string ends with given suffix. */
+bool
+ends_with(const char *str, const char *suffix)
+{
+ size_t suffix_len = strlen(suffix);
+ size_t str_len = strlen(str);
+ return(str_len >= suffix_len
+ && strcmp(str + str_len - suffix_len, suffix) == 0);
+}
+
+static bool starts_with(const char *str, const char *prefix)
+{
+ return strncmp(str, prefix, strlen(prefix)) == 0;
+}
+
+/************************************************************************
+Create directories recursively.
+@return 0 if directories created successfully. */
+static
+int
+mkdirp(const char *pathname, int Flags, myf MyFlags)
+{
+ char parent[PATH_MAX], *p;
+
+ /* make a parent directory path */
+ strncpy(parent, pathname, sizeof(parent));
+ parent[sizeof(parent) - 1] = 0;
+
+ for (p = parent + strlen(parent);
+ !is_path_separator(*p) && p != parent; p--);
+
+ *p = 0;
+
+ /* try to make parent directory */
+ if (p != parent && mkdirp(parent, Flags, MyFlags) != 0) {
+ return(-1);
+ }
+
+ /* make this one if parent has been made */
+ if (my_mkdir(pathname, Flags, MyFlags) == 0) {
+ return(0);
+ }
+
+ /* if it already exists that is fine */
+ if (errno == EEXIST) {
+ return(0);
+ }
+
+ return(-1);
+}
+
+/************************************************************************
+Return true if first and second arguments are the same path. */
+bool
+equal_paths(const char *first, const char *second)
+{
+#ifdef HAVE_REALPATH
+ char real_first[PATH_MAX];
+ char real_second[PATH_MAX];
+
+ if (realpath(first, real_first) == NULL) {
+ return false;
+ }
+ if (realpath(second, real_second) == NULL) {
+ return false;
+ }
+
+ return (strcmp(real_first, real_second) == 0);
+#else
+ return strcmp(first, second) == 0;
+#endif
+}
+
+/************************************************************************
+Check if directory exists. Optionally create directory if doesn't
+exist.
+@return true if directory exists and if it was created successfully. */
+bool
+directory_exists(const char *dir, bool create)
+{
+ os_file_dir_t os_dir;
+ MY_STAT stat_arg;
+ char errbuf[MYSYS_STRERROR_SIZE];
+
+ if (my_stat(dir, &stat_arg, MYF(0)) == NULL) {
+
+ if (!create) {
+ return(false);
+ }
+
+ if (mkdirp(dir, 0777, MYF(0)) < 0) {
+ my_strerror(errbuf, sizeof(errbuf), my_errno);
+ msg("Can not create directory %s: %s\n", dir, errbuf);
+ return(false);
+
+ }
+ }
+
+ /* could be symlink */
+ os_dir = os_file_opendir(dir, FALSE);
+
+ if (os_dir == NULL) {
+ my_strerror(errbuf, sizeof(errbuf), my_errno);
+ msg("Can not open directory %s: %s\n", dir,
+ errbuf);
+
+ return(false);
+ }
+
+ os_file_closedir(os_dir);
+
+ return(true);
+}
+
+/************************************************************************
+Check that directory exists and it is empty. */
+static
+bool
+directory_exists_and_empty(const char *dir, const char *comment)
+{
+ os_file_dir_t os_dir;
+ dberr_t err;
+ os_file_stat_t info;
+ bool empty;
+
+ if (!directory_exists(dir, true)) {
+ return(false);
+ }
+
+ os_dir = os_file_opendir(dir, FALSE);
+
+ if (os_dir == NULL) {
+ msg("%s can not open directory %s\n", comment, dir);
+ return(false);
+ }
+
+ empty = (fil_file_readdir_next_file(&err, dir, os_dir, &info) != 0);
+
+ os_file_closedir(os_dir);
+
+ if (!empty) {
+ msg("%s directory %s is not empty!\n", comment, dir);
+ }
+
+ return(empty);
+}
+
+
+/************************************************************************
+Check if file name ends with given set of suffixes.
+@return true if it does. */
+static
+bool
+filename_matches(const char *filename, const char **ext_list)
+{
+ const char **ext;
+
+ for (ext = ext_list; *ext; ext++) {
+ if (ends_with(filename, *ext)) {
+ return(true);
+ }
+ }
+
+ return(false);
+}
+
+
+/************************************************************************
+Copy data file for backup. Also check if it is allowed to copy by
+comparing its name to the list of known data file types and checking
+if passes the rules for partial backup.
+@return true if file backed up or skipped successfully. */
+static
+bool
+datafile_copy_backup(const char *filepath, uint thread_n)
+{
+ const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI",
+ "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par",
+ NULL};
+
+ /* Get the name and the path for the tablespace. node->name always
+ contains the path (which may be absolute for remote tablespaces in
+ 5.6+). space->name contains the tablespace name in the form
+ "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a
+ multi-node shared tablespace, space->name contains the name of the first
+ node, but that's irrelevant, since we only need node_name to match them
+ against filters, and the shared tablespace is always copied regardless
+ of the filters value. */
+
+ if (check_if_skip_table(filepath)) {
+ msg_ts("[%02u] Skipping %s.\n", thread_n, filepath);
+ return(true);
+ }
+
+ if (filename_matches(filepath, ext_list)) {
+ return copy_file(ds_data, filepath, filepath, thread_n);
+ }
+
+ return(true);
+}
+
+
+/************************************************************************
+Same as datafile_copy_backup, but put file name into the list for
+rsync command. */
+static
+bool
+datafile_rsync_backup(const char *filepath, bool save_to_list, FILE *f)
+{
+ const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI",
+ "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par",
+ NULL};
+
+ /* Get the name and the path for the tablespace. node->name always
+ contains the path (which may be absolute for remote tablespaces in
+ 5.6+). space->name contains the tablespace name in the form
+ "./database/table.ibd" (in 5.5-) or "database/table" (in 5.6+). For a
+ multi-node shared tablespace, space->name contains the name of the first
+ node, but that's irrelevant, since we only need node_name to match them
+ against filters, and the shared tablespace is always copied regardless
+ of the filters value. */
+
+ if (check_if_skip_table(filepath)) {
+ return(true);
+ }
+
+ if (filename_matches(filepath, ext_list)) {
+ fprintf(f, "%s\n", filepath);
+ if (save_to_list) {
+ rsync_list.insert(filepath);
+ }
+ }
+
+ return(true);
+}
+
+
+static
+bool
+backup_file_vprintf(const char *filename, const char *fmt, va_list ap)
+{
+ ds_file_t *dstfile = NULL;
+ MY_STAT stat; /* unused for now */
+ char *buf = 0;
+ int buf_len;
+ const char *action;
+
+ memset(&stat, 0, sizeof(stat));
+
+ buf_len = vasprintf(&buf, fmt, ap);
+
+ stat.st_size = buf_len;
+ stat.st_mtime = my_time(0);
+
+ dstfile = ds_open(ds_data, filename, &stat);
+ if (dstfile == NULL) {
+ msg("[%02u] error: "
+ "cannot open the destination stream for %s\n",
+ 0, filename);
+ goto error;
+ }
+
+ action = xb_get_copy_action("Writing");
+ msg_ts("[%02u] %s %s\n", 0, action, filename);
+
+ if (buf_len == -1) {
+ goto error;
+ }
+
+ if (ds_write(dstfile, buf, buf_len)) {
+ goto error;
+ }
+
+ /* close */
+ msg_ts("[%02u] ...done\n", 0);
+ free(buf);
+
+ if (ds_close(dstfile)) {
+ goto error_close;
+ }
+
+ return(true);
+
+error:
+ free(buf);
+ if (dstfile != NULL) {
+ ds_close(dstfile);
+ }
+
+error_close:
+ msg("[%02u] Error: backup file failed.\n", 0);
+ return(false); /*ERROR*/
+}
+
+
+bool
+backup_file_printf(const char *filename, const char *fmt, ...)
+{
+ bool result;
+ va_list ap;
+
+ va_start(ap, fmt);
+
+ result = backup_file_vprintf(filename, fmt, ap);
+
+ va_end(ap);
+
+ return(result);
+}
+
+static
+bool
+run_data_threads(datadir_iter_t *it, os_thread_func_t func, uint n)
+{
+ datadir_thread_ctxt_t *data_threads;
+ uint i, count;
+ pthread_mutex_t count_mutex;
+ bool ret;
+
+ data_threads = (datadir_thread_ctxt_t*)
+ malloc(sizeof(datadir_thread_ctxt_t) * n);
+
+ pthread_mutex_init(&count_mutex, NULL);
+ count = n;
+
+ for (i = 0; i < n; i++) {
+ data_threads[i].it = it;
+ data_threads[i].n_thread = i + 1;
+ data_threads[i].count = &count;
+ data_threads[i].count_mutex = count_mutex;
+ os_thread_create(func, data_threads + i, &data_threads[i].id);
+ }
+
+ /* Wait for threads to exit */
+ while (1) {
+ os_thread_sleep(100000);
+ pthread_mutex_lock(&count_mutex);
+ if (count == 0) {
+ pthread_mutex_unlock(&count_mutex);
+ break;
+ }
+ pthread_mutex_unlock(&count_mutex);
+ }
+
+ pthread_mutex_destroy(&count_mutex);
+
+ ret = true;
+ for (i = 0; i < n; i++) {
+ ret = data_threads[i].ret && ret;
+ if (!data_threads[i].ret) {
+ msg("Error: thread %u failed.\n", i);
+ }
+ }
+
+ free(data_threads);
+
+ return(ret);
+}
+
+
+/************************************************************************
+Copy file for backup/restore.
+@return true in case of success. */
+bool
+copy_file(ds_ctxt_t *datasink,
+ const char *src_file_path,
+ const char *dst_file_path,
+ uint thread_n)
+{
+ char dst_name[FN_REFLEN];
+ ds_file_t *dstfile = NULL;
+ datafile_cur_t cursor;
+ xb_fil_cur_result_t res;
+
+ if (!datafile_open(src_file_path, &cursor, thread_n)) {
+ goto error_close;
+ }
+
+ strncpy(dst_name, cursor.rel_path, sizeof(dst_name));
+
+ dstfile = ds_open(datasink, trim_dotslash(dst_file_path),
+ &cursor.statinfo);
+ if (dstfile == NULL) {
+ msg("[%02u] error: "
+ "cannot open the destination stream for %s\n",
+ thread_n, dst_name);
+ goto error;
+ }
+
+ msg_ts("[%02u] %s %s to %s\n",
+ thread_n, xb_get_copy_action(), src_file_path, dstfile->path);
+
+ /* The main copy loop */
+ while ((res = datafile_read(&cursor)) == XB_FIL_CUR_SUCCESS) {
+
+ if (ds_write(dstfile, cursor.buf, cursor.buf_read)) {
+ goto error;
+ }
+ }
+
+ if (res == XB_FIL_CUR_ERROR) {
+ goto error;
+ }
+
+ /* close */
+ msg_ts("[%02u] ...done\n", thread_n);
+ datafile_close(&cursor);
+ if (ds_close(dstfile)) {
+ goto error_close;
+ }
+ return(true);
+
+error:
+ datafile_close(&cursor);
+ if (dstfile != NULL) {
+ ds_close(dstfile);
+ }
+
+error_close:
+ msg("[%02u] Error: copy_file() failed.\n", thread_n);
+ return(false); /*ERROR*/
+}
+
+
+/************************************************************************
+Try to move file by renaming it. If source and destination are on
+different devices fall back to copy and unlink.
+@return true in case of success. */
+static
+bool
+move_file(ds_ctxt_t *datasink,
+ const char *src_file_path,
+ const char *dst_file_path,
+ const char *dst_dir, uint thread_n)
+{
+ char errbuf[MYSYS_STRERROR_SIZE];
+ char dst_file_path_abs[FN_REFLEN];
+ char dst_dir_abs[FN_REFLEN];
+ size_t dirname_length;
+
+ ut_snprintf(dst_file_path_abs, sizeof(dst_file_path_abs),
+ "%s/%s", dst_dir, dst_file_path);
+
+ dirname_part(dst_dir_abs, dst_file_path_abs, &dirname_length);
+
+ if (!directory_exists(dst_dir_abs, true)) {
+ return(false);
+ }
+
+ if (file_exists(dst_file_path_abs)) {
+ msg("Error: Move file %s to %s failed: Destination "
+ "file exists\n",
+ src_file_path, dst_file_path_abs);
+ return(false);
+ }
+
+ msg_ts("[%02u] Moving %s to %s\n",
+ thread_n, src_file_path, dst_file_path_abs);
+
+ if (my_rename(src_file_path, dst_file_path_abs, MYF(0)) != 0) {
+ if (my_errno == EXDEV) {
+ bool ret;
+ ret = copy_file(datasink, src_file_path,
+ dst_file_path, thread_n);
+ msg_ts("[%02u] Removing %s\n", thread_n, src_file_path);
+ if (unlink(src_file_path) != 0) {
+ my_strerror(errbuf, sizeof(errbuf), errno);
+ msg("Error: unlink %s failed: %s\n",
+ src_file_path,
+ errbuf);
+ }
+ return(ret);
+ }
+ my_strerror(errbuf, sizeof(errbuf), my_errno);
+ msg("Can not move file %s to %s: %s\n",
+ src_file_path, dst_file_path_abs,
+ errbuf);
+ return(false);
+ }
+
+ msg_ts("[%02u] ...done\n", thread_n);
+
+ return(true);
+}
+
+
+/************************************************************************
+Read link from .isl file if any and store it in the global map associated
+with given tablespace. */
+static
+void
+read_link_file(const char *ibd_filepath, const char *link_filepath)
+{
+ char *filepath= NULL;
+
+ FILE *file = fopen(link_filepath, "r+b");
+ if (file) {
+ filepath = static_cast(malloc(OS_FILE_MAX_PATH));
+
+ os_file_read_string(file, filepath, OS_FILE_MAX_PATH);
+ fclose(file);
+
+ if (strlen(filepath)) {
+ /* Trim whitespace from end of filepath */
+ ulint lastch = strlen(filepath) - 1;
+ while (lastch > 4 && filepath[lastch] <= 0x20) {
+ filepath[lastch--] = 0x00;
+ }
+ os_normalize_path(filepath);
+ }
+
+ tablespace_locations[ibd_filepath] = filepath;
+ }
+ free(filepath);
+}
+
+
+/************************************************************************
+Return the location of given .ibd if it was previously read
+from .isl file.
+@return NULL or destination .ibd file path. */
+static
+const char *
+tablespace_filepath(const char *ibd_filepath)
+{
+ std::map::iterator it;
+
+ it = tablespace_locations.find(ibd_filepath);
+
+ if (it != tablespace_locations.end()) {
+ return it->second.c_str();
+ }
+
+ return NULL;
+}
+
+
+/************************************************************************
+Copy or move file depending on current mode.
+@return true in case of success. */
+static
+bool
+copy_or_move_file(const char *src_file_path,
+ const char *dst_file_path,
+ const char *dst_dir,
+ uint thread_n)
+{
+ ds_ctxt_t *datasink = ds_data; /* copy to datadir by default */
+ char filedir[FN_REFLEN];
+ size_t filedir_len;
+ bool ret;
+
+ /* read the link from .isl file */
+ if (ends_with(src_file_path, ".isl")) {
+ char *ibd_filepath;
+
+ ibd_filepath = strdup(src_file_path);
+ strcpy(ibd_filepath + strlen(ibd_filepath) - 3, "ibd");
+
+ read_link_file(ibd_filepath, src_file_path);
+
+ free(ibd_filepath);
+ }
+
+ /* check if there is .isl file */
+ if (ends_with(src_file_path, ".ibd")) {
+ char *link_filepath;
+ const char *filepath;
+
+ link_filepath = strdup(src_file_path);
+ strcpy(link_filepath + strlen(link_filepath) - 3, "isl");
+
+ read_link_file(src_file_path, link_filepath);
+
+ filepath = tablespace_filepath(src_file_path);
+
+ if (filepath != NULL) {
+ dirname_part(filedir, filepath, &filedir_len);
+
+ dst_file_path = filepath + filedir_len;
+ dst_dir = filedir;
+
+ if (!directory_exists(dst_dir, true)) {
+ ret = false;
+ goto cleanup;
+ }
+
+ datasink = ds_create(dst_dir, DS_TYPE_LOCAL);
+ }
+
+ free(link_filepath);
+ }
+
+ ret = (xtrabackup_copy_back ?
+ copy_file(datasink, src_file_path, dst_file_path, thread_n) :
+ move_file(datasink, src_file_path, dst_file_path,
+ dst_dir, thread_n));
+
+cleanup:
+
+ if (datasink != ds_data) {
+ ds_destroy(datasink);
+ }
+
+ return(ret);
+}
+
+
+
+
+bool
+backup_files(const char *from, bool prep_mode)
+{
+ char rsync_tmpfile_name[FN_REFLEN];
+ FILE *rsync_tmpfile = NULL;
+ datadir_iter_t *it;
+ datadir_node_t node;
+ bool ret = true;
+
+ if (prep_mode && !opt_rsync) {
+ return(true);
+ }
+
+ if (opt_rsync) {
+ snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name),
+ "%s/%s%d", opt_mysql_tmpdir,
+ "xtrabackup_rsyncfiles_pass",
+ prep_mode ? 1 : 2);
+ rsync_tmpfile = fopen(rsync_tmpfile_name, "w");
+ if (rsync_tmpfile == NULL) {
+ msg("Error: can't create file %s\n",
+ rsync_tmpfile_name);
+ return(false);
+ }
+ }
+
+ msg_ts("Starting %s non-InnoDB tables and files\n",
+ prep_mode ? "prep copy of" : "to backup");
+
+ datadir_node_init(&node);
+ it = datadir_iter_new(from);
+
+ while (datadir_iter_next(it, &node)) {
+
+ if (!node.is_empty_dir) {
+ if (opt_rsync) {
+ ret = datafile_rsync_backup(node.filepath,
+ !prep_mode, rsync_tmpfile);
+ } else {
+ ret = datafile_copy_backup(node.filepath, 1);
+ }
+ if (!ret) {
+ msg("Failed to copy file %s\n", node.filepath);
+ goto out;
+ }
+ } else if (!prep_mode) {
+ /* backup fake file into empty directory */
+ char path[FN_REFLEN];
+ ut_snprintf(path, sizeof(path),
+ "%s/db.opt", node.filepath);
+ if (!(ret = backup_file_printf(
+ trim_dotslash(path), "%s", ""))) {
+ msg("Failed to create file %s\n", path);
+ goto out;
+ }
+ }
+ }
+
+ if (opt_rsync) {
+ std::stringstream cmd;
+ int err;
+
+ if (buffer_pool_filename && file_exists(buffer_pool_filename)) {
+ fprintf(rsync_tmpfile, "%s\n", buffer_pool_filename);
+ rsync_list.insert(buffer_pool_filename);
+ }
+ if (file_exists("ib_lru_dump")) {
+ fprintf(rsync_tmpfile, "%s\n", "ib_lru_dump");
+ rsync_list.insert("ib_lru_dump");
+ }
+
+ fclose(rsync_tmpfile);
+ rsync_tmpfile = NULL;
+
+ cmd << "rsync -t . --files-from=" << rsync_tmpfile_name
+ << " " << xtrabackup_target_dir;
+
+ msg_ts("Starting rsync as: %s\n", cmd.str().c_str());
+ if ((err = system(cmd.str().c_str()) && !prep_mode) != 0) {
+ msg_ts("Error: rsync failed with error code %d\n", err);
+ ret = false;
+ goto out;
+ }
+ msg_ts("rsync finished successfully.\n");
+
+ if (!prep_mode && !opt_no_lock) {
+ char path[FN_REFLEN];
+ char dst_path[FN_REFLEN];
+ char *newline;
+
+ /* Remove files that have been removed between first and
+ second passes. Cannot use "rsync --delete" because it
+ does not work with --files-from. */
+ snprintf(rsync_tmpfile_name, sizeof(rsync_tmpfile_name),
+ "%s/%s", opt_mysql_tmpdir,
+ "xtrabackup_rsyncfiles_pass1");
+
+ rsync_tmpfile = fopen(rsync_tmpfile_name, "r");
+ if (rsync_tmpfile == NULL) {
+ msg("Error: can't open file %s\n",
+ rsync_tmpfile_name);
+ return(false);
+ }
+
+ while (fgets(path, sizeof(path), rsync_tmpfile)) {
+
+ newline = strchr(path, '\n');
+ if (newline) {
+ *newline = 0;
+ }
+ if (rsync_list.count(path) < 1) {
+ snprintf(dst_path, sizeof(dst_path),
+ "%s/%s", xtrabackup_target_dir,
+ path);
+ msg_ts("Removing %s\n", dst_path);
+ unlink(dst_path);
+ }
+ }
+
+ fclose(rsync_tmpfile);
+ rsync_tmpfile = NULL;
+ }
+ }
+
+ msg_ts("Finished %s non-InnoDB tables and files\n",
+ prep_mode ? "a prep copy of" : "backing up");
+
+out:
+ datadir_iter_free(it);
+ datadir_node_free(&node);
+
+ if (rsync_tmpfile != NULL) {
+ fclose(rsync_tmpfile);
+ }
+
+ return(ret);
+}
+
+/** Start --backup */
+bool backup_start()
+{
+ if (!opt_no_lock) {
+ if (opt_safe_slave_backup) {
+ if (!wait_for_safe_slave(mysql_connection)) {
+ return(false);
+ }
+ }
+
+ if (!backup_files(fil_path_to_mysql_datadir, true)) {
+ return(false);
+ }
+
+ history_lock_time = time(NULL);
+
+ if (!lock_tables(mysql_connection)) {
+ return(false);
+ }
+ }
+
+ if (!backup_files(fil_path_to_mysql_datadir, false)) {
+ return(false);
+ }
+
+ if (!backup_files_from_datadir(fil_path_to_mysql_datadir)) {
+ return false;
+ }
+
+ // There is no need to stop slave thread before coping non-Innodb data when
+ // --no-lock option is used because --no-lock option requires that no DDL or
+ // DML to non-transaction tables can occur.
+ if (opt_no_lock) {
+ if (opt_safe_slave_backup) {
+ if (!wait_for_safe_slave(mysql_connection)) {
+ return(false);
+ }
+ }
+ }
+
+ if (opt_slave_info) {
+ lock_binlog_maybe(mysql_connection);
+
+ if (!write_slave_info(mysql_connection)) {
+ return(false);
+ }
+ }
+
+ /* The only reason why Galera/binlog info is written before
+ wait_for_ibbackup_log_copy_finish() is that after that call the xtrabackup
+ binary will start streamig a temporary copy of REDO log to stdout and
+ thus, any streaming from innobackupex would interfere. The only way to
+ avoid that is to have a single process, i.e. merge innobackupex and
+ xtrabackup. */
+ if (opt_galera_info) {
+ if (!write_galera_info(mysql_connection)) {
+ return(false);
+ }
+ write_current_binlog_file(mysql_connection);
+ }
+
+ if (opt_binlog_info == BINLOG_INFO_ON) {
+
+ lock_binlog_maybe(mysql_connection);
+ write_binlog_info(mysql_connection);
+ }
+
+ if (have_flush_engine_logs) {
+ msg_ts("Executing FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS...\n");
+ xb_mysql_query(mysql_connection,
+ "FLUSH NO_WRITE_TO_BINLOG ENGINE LOGS", false);
+ }
+
+ return(true);
+}
+
+/** Release resources after backup_start() */
+void backup_release()
+{
+ /* release all locks */
+ if (!opt_no_lock) {
+ unlock_all(mysql_connection);
+ history_lock_time = 0;
+ } else {
+ history_lock_time = time(NULL) - history_lock_time;
+ }
+
+ if (opt_safe_slave_backup && sql_thread_started) {
+ msg("Starting slave SQL thread\n");
+ xb_mysql_query(mysql_connection,
+ "START SLAVE SQL_THREAD", false);
+ }
+}
+
+/** Finish after backup_start() and backup_release() */
+bool backup_finish()
+{
+ /* Copy buffer pool dump or LRU dump */
+ if (!opt_rsync) {
+ if (buffer_pool_filename && file_exists(buffer_pool_filename)) {
+ const char *dst_name;
+
+ dst_name = trim_dotslash(buffer_pool_filename);
+ copy_file(ds_data, buffer_pool_filename, dst_name, 0);
+ }
+ if (file_exists("ib_lru_dump")) {
+ copy_file(ds_data, "ib_lru_dump", "ib_lru_dump", 0);
+ }
+ }
+
+ msg_ts("Backup created in directory '%s'\n", xtrabackup_target_dir);
+ if (mysql_binlog_position != NULL) {
+ msg("MySQL binlog position: %s\n", mysql_binlog_position);
+ }
+ if (mysql_slave_position && opt_slave_info) {
+ msg("MySQL slave binlog position: %s\n",
+ mysql_slave_position);
+ }
+
+ if (!write_backup_config_file()) {
+ return(false);
+ }
+
+ if (!write_xtrabackup_info(mysql_connection)) {
+ return(false);
+ }
+
+
+
+ return(true);
+}
+
+bool
+ibx_copy_incremental_over_full()
+{
+ const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI",
+ "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par",
+ NULL};
+ const char *sup_files[] = {"xtrabackup_binlog_info",
+ "xtrabackup_galera_info",
+ "xtrabackup_slave_info",
+ "xtrabackup_info",
+ "ib_lru_dump",
+ NULL};
+ datadir_iter_t *it = NULL;
+ datadir_node_t node;
+ bool ret = true;
+ char path[FN_REFLEN];
+ int i;
+
+ datadir_node_init(&node);
+
+ /* If we were applying an incremental change set, we need to make
+ sure non-InnoDB files and xtrabackup_* metainfo files are copied
+ to the full backup directory. */
+
+ if (xtrabackup_incremental) {
+
+ ds_data = ds_create(xtrabackup_target_dir, DS_TYPE_LOCAL);
+
+ it = datadir_iter_new(xtrabackup_incremental_dir);
+
+ while (datadir_iter_next(it, &node)) {
+
+ /* copy only non-innodb files */
+
+ if (node.is_empty_dir
+ || !filename_matches(node.filepath, ext_list)) {
+ continue;
+ }
+
+ if (file_exists(node.filepath_rel)) {
+ unlink(node.filepath_rel);
+ }
+
+ if (!(ret = copy_file(ds_data, node.filepath,
+ node.filepath_rel, 1))) {
+ msg("Failed to copy file %s\n",
+ node.filepath);
+ goto cleanup;
+ }
+ }
+
+ /* copy buffer pool dump */
+ if (innobase_buffer_pool_filename) {
+ const char *src_name;
+
+ src_name = trim_dotslash(innobase_buffer_pool_filename);
+
+ snprintf(path, sizeof(path), "%s/%s",
+ xtrabackup_incremental_dir,
+ src_name);
+
+ if (file_exists(path)) {
+ copy_file(ds_data, path,
+ innobase_buffer_pool_filename, 0);
+ }
+ }
+
+ /* copy supplementary files */
+
+ for (i = 0; sup_files[i]; i++) {
+ snprintf(path, sizeof(path), "%s/%s",
+ xtrabackup_incremental_dir,
+ sup_files[i]);
+
+ if (file_exists(path))
+ {
+ if (file_exists(sup_files[i])) {
+ unlink(sup_files[i]);
+ }
+ copy_file(ds_data, path, sup_files[i], 0);
+ }
+ }
+
+ }
+
+cleanup:
+ if (it != NULL) {
+ datadir_iter_free(it);
+ }
+
+ if (ds_data != NULL) {
+ ds_destroy(ds_data);
+ }
+
+ datadir_node_free(&node);
+
+ return(ret);
+}
+
+bool
+ibx_cleanup_full_backup()
+{
+ const char *ext_list[] = {"delta", "meta", "ibd", NULL};
+ datadir_iter_t *it = NULL;
+ datadir_node_t node;
+ bool ret = true;
+
+ datadir_node_init(&node);
+
+ /* If we are applying an incremental change set, we need to make
+ sure non-InnoDB files are cleaned up from full backup dir before
+ we copy files from incremental dir. */
+
+ it = datadir_iter_new(xtrabackup_target_dir);
+
+ while (datadir_iter_next(it, &node)) {
+
+ if (node.is_empty_dir) {
+#ifdef _WIN32
+ DeleteFile(node.filepath);
+#else
+ rmdir(node.filepath);
+#endif
+ }
+
+ if (xtrabackup_incremental && !node.is_empty_dir
+ && !filename_matches(node.filepath, ext_list)) {
+ unlink(node.filepath);
+ }
+ }
+
+ datadir_iter_free(it);
+
+ datadir_node_free(&node);
+
+ return(ret);
+}
+
+bool
+apply_log_finish()
+{
+ if (!ibx_cleanup_full_backup()
+ || !ibx_copy_incremental_over_full()) {
+ return(false);
+ }
+
+ return(true);
+}
+
+bool
+copy_back()
+{
+ bool ret;
+ datadir_iter_t *it = NULL;
+ datadir_node_t node;
+ char *dst_dir;
+
+ memset(&node, 0, sizeof(node));
+
+ if (!opt_force_non_empty_dirs) {
+ if (!directory_exists_and_empty(mysql_data_home,
+ "Original data")) {
+ return(false);
+ }
+ } else {
+ if (!directory_exists(mysql_data_home, true)) {
+ return(false);
+ }
+ }
+ if (srv_undo_dir && *srv_undo_dir
+ && !directory_exists(srv_undo_dir, true)) {
+ return(false);
+ }
+ if (innobase_data_home_dir && *innobase_data_home_dir
+ && !directory_exists(innobase_data_home_dir, true)) {
+ return(false);
+ }
+ if (srv_log_group_home_dir && *srv_log_group_home_dir
+ && !directory_exists(srv_log_group_home_dir, true)) {
+ return(false);
+ }
+
+ /* cd to backup directory */
+ if (my_setwd(xtrabackup_target_dir, MYF(MY_WME)))
+ {
+ msg("cannot my_setwd %s\n", xtrabackup_target_dir);
+ return(false);
+ }
+
+ /* parse data file path */
+
+ if (!innobase_data_file_path) {
+ innobase_data_file_path = (char*) "ibdata1:10M:autoextend";
+ }
+
+ srv_sys_space.set_path(".");
+
+ if (!srv_sys_space.parse_params(innobase_data_file_path, true)) {
+ msg("syntax error in innodb_data_file_path\n");
+ return(false);
+ }
+
+ srv_max_n_threads = 1000;
+ sync_check_init();
+ ut_crc32_init();
+
+ /* copy undo tablespaces */
+ if (srv_undo_tablespaces > 0) {
+
+ dst_dir = (srv_undo_dir && *srv_undo_dir)
+ ? srv_undo_dir : mysql_data_home;
+
+ ds_data = ds_create(dst_dir, DS_TYPE_LOCAL);
+
+ for (ulong i = 1; i <= srv_undo_tablespaces; i++) {
+ char filename[20];
+ sprintf(filename, "undo%03lu", i);
+ if (!(ret = copy_or_move_file(filename, filename,
+ dst_dir, 1))) {
+ goto cleanup;
+ }
+ }
+
+ ds_destroy(ds_data);
+ ds_data = NULL;
+ }
+
+ dst_dir = (srv_log_group_home_dir && *srv_log_group_home_dir)
+ ? srv_log_group_home_dir : mysql_data_home;
+
+ /* --backup generates a single ib_logfile0, which we must copy
+ if it exists. */
+
+ ds_data = ds_create(dst_dir, DS_TYPE_LOCAL);
+ if (!file_exists("ib_logfile0")) {
+ /* After completed --prepare, redo log files are redundant.
+ We must delete any redo logs at the destination, so that
+ the database will not jump to a different log sequence number
+ (LSN). */
+
+ for (uint i = 0; i <= SRV_N_LOG_FILES_MAX + 1; i++) {
+ char filename[FN_REFLEN];
+ snprintf(filename, sizeof filename, "%s/ib_logfile%u",
+ dst_dir, i);
+ unlink(filename);
+ }
+ } else if (!(ret = copy_or_move_file("ib_logfile0", "ib_logfile0",
+ dst_dir, 1))) {
+ goto cleanup;
+ }
+ ds_destroy(ds_data);
+
+ /* copy innodb system tablespace(s) */
+
+ dst_dir = (innobase_data_home_dir && *innobase_data_home_dir)
+ ? innobase_data_home_dir : mysql_data_home;
+
+ ds_data = ds_create(dst_dir, DS_TYPE_LOCAL);
+
+ for (Tablespace::const_iterator iter(srv_sys_space.begin()),
+ end(srv_sys_space.end());
+ iter != end;
+ ++iter) {
+ const char *filename = base_name(iter->name());
+
+ if (!(ret = copy_or_move_file(filename, iter->name(),
+ dst_dir, 1))) {
+ goto cleanup;
+ }
+ }
+
+ ds_destroy(ds_data);
+
+ /* copy the rest of tablespaces */
+ ds_data = ds_create(mysql_data_home, DS_TYPE_LOCAL);
+
+ it = datadir_iter_new(".", false);
+
+ datadir_node_init(&node);
+
+ while (datadir_iter_next(it, &node)) {
+ const char *ext_list[] = {"backup-my.cnf",
+ "xtrabackup_binary", "xtrabackup_binlog_info",
+ "xtrabackup_checkpoints", ".qp", ".pmap", ".tmp",
+ NULL};
+ const char *filename;
+ char c_tmp;
+ int i_tmp;
+ bool is_ibdata_file;
+
+ /* create empty directories */
+ if (node.is_empty_dir) {
+ char path[FN_REFLEN];
+
+ snprintf(path, sizeof(path), "%s/%s",
+ mysql_data_home, node.filepath_rel);
+
+ msg_ts("[%02u] Creating directory %s\n", 1, path);
+
+ if (mkdirp(path, 0777, MYF(0)) < 0) {
+ char errbuf[MYSYS_STRERROR_SIZE];
+ my_strerror(errbuf, sizeof(errbuf), my_errno);
+ msg("Can not create directory %s: %s\n",
+ path, errbuf);
+ ret = false;
+
+ goto cleanup;
+
+ }
+
+ msg_ts("[%02u] ...done.", 1);
+
+ continue;
+ }
+
+ filename = base_name(node.filepath);
+
+ /* skip .qp files */
+ if (filename_matches(filename, ext_list)) {
+ continue;
+ }
+
+ /* skip undo tablespaces */
+ if (sscanf(filename, "undo%d%c", &i_tmp, &c_tmp) == 1) {
+ continue;
+ }
+
+ /* skip the redo log (it was already copied) */
+ if (!strcmp(filename, "ib_logfile0")) {
+ continue;
+ }
+
+ /* skip innodb data files */
+ is_ibdata_file = false;
+ for (Tablespace::const_iterator iter(srv_sys_space.begin()),
+ end(srv_sys_space.end()); iter != end; ++iter) {
+ if (strcmp(iter->name(), filename) == 0) {
+ is_ibdata_file = true;
+ break;
+ }
+ }
+ if (is_ibdata_file) {
+ continue;
+ }
+
+ if (!(ret = copy_or_move_file(node.filepath, node.filepath_rel,
+ mysql_data_home, 1))) {
+ goto cleanup;
+ }
+ }
+
+ /* copy buufer pool dump */
+
+ if (innobase_buffer_pool_filename) {
+ const char *src_name;
+ char path[FN_REFLEN];
+
+ src_name = trim_dotslash(innobase_buffer_pool_filename);
+
+ snprintf(path, sizeof(path), "%s/%s",
+ mysql_data_home,
+ src_name);
+
+ /* could be already copied with other files
+ from data directory */
+ if (file_exists(src_name) &&
+ !file_exists(innobase_buffer_pool_filename)) {
+ copy_or_move_file(src_name,
+ innobase_buffer_pool_filename,
+ mysql_data_home, 0);
+ }
+ }
+
+cleanup:
+ if (it != NULL) {
+ datadir_iter_free(it);
+ }
+
+ datadir_node_free(&node);
+
+ if (ds_data != NULL) {
+ ds_destroy(ds_data);
+ }
+
+ ds_data = NULL;
+
+ sync_check_close();
+ return(ret);
+}
+
+bool
+decrypt_decompress_file(const char *filepath, uint thread_n)
+{
+ std::stringstream cmd, message;
+ char *dest_filepath = strdup(filepath);
+ bool needs_action = false;
+
+ cmd << IF_WIN("type ","cat ") << filepath;
+
+ if (opt_decompress
+ && ends_with(filepath, ".qp")) {
+ cmd << " | qpress -dio ";
+ dest_filepath[strlen(dest_filepath) - 3] = 0;
+ if (needs_action) {
+ message << " and ";
+ }
+ message << "decompressing";
+ needs_action = true;
+ }
+
+ cmd << " > " << dest_filepath;
+ message << " " << filepath;
+
+ free(dest_filepath);
+
+ if (needs_action) {
+
+ msg_ts("[%02u] %s\n", thread_n, message.str().c_str());
+
+ if (system(cmd.str().c_str()) != 0) {
+ return(false);
+ }
+ }
+
+ return(true);
+}
+
+static
+os_thread_ret_t STDCALL
+decrypt_decompress_thread_func(void *arg)
+{
+ bool ret = true;
+ datadir_node_t node;
+ datadir_thread_ctxt_t *ctxt = (datadir_thread_ctxt_t *)(arg);
+
+ datadir_node_init(&node);
+
+ while (datadir_iter_next(ctxt->it, &node)) {
+
+ /* skip empty directories in backup */
+ if (node.is_empty_dir) {
+ continue;
+ }
+
+ if (!ends_with(node.filepath, ".qp")) {
+ continue;
+ }
+
+ if (!(ret = decrypt_decompress_file(node.filepath,
+ ctxt->n_thread))) {
+ goto cleanup;
+ }
+ }
+
+cleanup:
+
+ datadir_node_free(&node);
+
+ pthread_mutex_lock(&ctxt->count_mutex);
+ --(*ctxt->count);
+ pthread_mutex_unlock(&ctxt->count_mutex);
+
+ ctxt->ret = ret;
+
+ os_thread_exit();
+ OS_THREAD_DUMMY_RETURN;
+}
+
+bool
+decrypt_decompress()
+{
+ bool ret;
+ datadir_iter_t *it = NULL;
+
+ srv_max_n_threads = 1000;
+ sync_check_init();
+
+ /* cd to backup directory */
+ if (my_setwd(xtrabackup_target_dir, MYF(MY_WME)))
+ {
+ msg("cannot my_setwd %s\n", xtrabackup_target_dir);
+ return(false);
+ }
+
+ /* copy the rest of tablespaces */
+ ds_data = ds_create(".", DS_TYPE_LOCAL);
+
+ it = datadir_iter_new(".", false);
+
+ ut_a(xtrabackup_parallel >= 0);
+
+ ret = run_data_threads(it, decrypt_decompress_thread_func,
+ xtrabackup_parallel ? xtrabackup_parallel : 1);
+
+ if (it != NULL) {
+ datadir_iter_free(it);
+ }
+
+ if (ds_data != NULL) {
+ ds_destroy(ds_data);
+ }
+
+ ds_data = NULL;
+
+ sync_check_close();
+
+ return(ret);
+}
+
+/*
+ Copy some files from top level datadir.
+ Do not copy the Innodb files (ibdata1, redo log files),
+ as this is done in a separate step.
+*/
+static bool backup_files_from_datadir(const char *dir_path)
+{
+ os_file_dir_t dir = os_file_opendir(dir_path, TRUE);
+ os_file_stat_t info;
+ bool ret = true;
+ while (os_file_readdir_next_file(dir_path, dir, &info) == 0) {
+
+ if (info.type != OS_FILE_TYPE_FILE)
+ continue;
+
+ const char *pname = strrchr(info.name, IF_WIN('\\', '/'));
+ if (!pname)
+ pname = info.name;
+
+ /* Copy aria log files, and aws keys for encryption plugins.*/
+ const char *prefixes[] = { "aria_log", "aws-kms-key" };
+ for (size_t i = 0; i < array_elements(prefixes); i++) {
+ if (starts_with(pname, prefixes[i])) {
+ ret = copy_file(ds_data, info.name, info.name, 1);
+ if (!ret) {
+ break;
+ }
+ }
+ }
+ }
+ os_file_closedir(dir);
+ return ret;
+}
diff --git a/extra/mariabackup/backup_copy.h b/extra/mariabackup/backup_copy.h
new file mode 100644
index 00000000000..fbc09eaded3
--- /dev/null
+++ b/extra/mariabackup/backup_copy.h
@@ -0,0 +1,51 @@
+
+#ifndef XTRABACKUP_BACKUP_COPY_H
+#define XTRABACKUP_BACKUP_COPY_H
+
+#include
+#include "datasink.h"
+
+/* special files */
+#define XTRABACKUP_SLAVE_INFO "xtrabackup_slave_info"
+#define XTRABACKUP_GALERA_INFO "xtrabackup_galera_info"
+#define XTRABACKUP_BINLOG_INFO "xtrabackup_binlog_info"
+#define XTRABACKUP_INFO "xtrabackup_info"
+
+extern bool binlog_locked;
+
+bool
+backup_file_printf(const char *filename, const char *fmt, ...)
+ ATTRIBUTE_FORMAT(printf, 2, 0);
+
+/************************************************************************
+Return true if first and second arguments are the same path. */
+bool
+equal_paths(const char *first, const char *second);
+
+/************************************************************************
+Copy file for backup/restore.
+@return true in case of success. */
+bool
+copy_file(ds_ctxt_t *datasink,
+ const char *src_file_path,
+ const char *dst_file_path,
+ uint thread_n);
+
+/** Start --backup */
+bool backup_start();
+/** Release resources after backup_start() */
+void backup_release();
+/** Finish after backup_start() and backup_release() */
+bool backup_finish();
+bool
+apply_log_finish();
+bool
+copy_back();
+bool
+decrypt_decompress();
+bool
+is_path_separator(char);
+bool
+directory_exists(const char *dir, bool create);
+
+#endif
diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc
new file mode 100644
index 00000000000..cebbbf3a28c
--- /dev/null
+++ b/extra/mariabackup/backup_mysql.cc
@@ -0,0 +1,1706 @@
+/******************************************************
+hot backup tool for InnoDB
+(c) 2009-2015 Percona LLC and/or its affiliates
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************
+
+This file incorporates work covered by the following copyright and
+permission notice:
+
+Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA
+
+*******************************************************/
+#define MYSQL_CLIENT
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include "common.h"
+#include "xtrabackup.h"
+#include "mysql_version.h"
+#include "backup_copy.h"
+#include "backup_mysql.h"
+#include "mysqld.h"
+#include "encryption_plugin.h"
+#include
+#include
+#include
+
+
+char *tool_name;
+char tool_args[2048];
+
+/* mysql flavor and version */
+mysql_flavor_t server_flavor = FLAVOR_UNKNOWN;
+unsigned long mysql_server_version = 0;
+
+/* server capabilities */
+bool have_changed_page_bitmaps = false;
+bool have_backup_locks = false;
+bool have_backup_safe_binlog_info = false;
+bool have_lock_wait_timeout = false;
+bool have_galera_enabled = false;
+bool have_flush_engine_logs = false;
+bool have_multi_threaded_slave = false;
+bool have_gtid_slave = false;
+
+/* Kill long selects */
+os_thread_id_t kill_query_thread_id;
+os_event_t kill_query_thread_started;
+os_event_t kill_query_thread_stopped;
+os_event_t kill_query_thread_stop;
+
+bool sql_thread_started = false;
+char *mysql_slave_position = NULL;
+char *mysql_binlog_position = NULL;
+char *buffer_pool_filename = NULL;
+
+/* History on server */
+time_t history_start_time;
+time_t history_end_time;
+time_t history_lock_time;
+
+MYSQL *mysql_connection;
+
+my_bool opt_ssl_verify_server_cert;
+
+MYSQL *
+xb_mysql_connect()
+{
+ MYSQL *connection = mysql_init(NULL);
+ char mysql_port_str[std::numeric_limits::digits10 + 3];
+
+ sprintf(mysql_port_str, "%d", opt_port);
+
+ if (connection == NULL) {
+ msg("Failed to init MySQL struct: %s.\n",
+ mysql_error(connection));
+ return(NULL);
+ }
+
+ if (!opt_secure_auth) {
+ mysql_options(connection, MYSQL_SECURE_AUTH,
+ (char *) &opt_secure_auth);
+ }
+
+ if (xb_plugin_dir && *xb_plugin_dir){
+ mysql_options(connection, MYSQL_PLUGIN_DIR, xb_plugin_dir);
+ }
+ mysql_options(connection, MYSQL_OPT_PROTOCOL, &opt_protocol);
+ mysql_options(connection,MYSQL_SET_CHARSET_NAME, "utf8");
+
+ msg_ts("Connecting to MySQL server host: %s, user: %s, password: %s, "
+ "port: %s, socket: %s\n", opt_host ? opt_host : "localhost",
+ opt_user ? opt_user : "not set",
+ opt_password ? "set" : "not set",
+ opt_port != 0 ? mysql_port_str : "not set",
+ opt_socket ? opt_socket : "not set");
+
+#ifdef HAVE_OPENSSL
+ if (opt_use_ssl)
+ {
+ mysql_ssl_set(connection, opt_ssl_key, opt_ssl_cert,
+ opt_ssl_ca, opt_ssl_capath,
+ opt_ssl_cipher);
+ mysql_options(connection, MYSQL_OPT_SSL_CRL, opt_ssl_crl);
+ mysql_options(connection, MYSQL_OPT_SSL_CRLPATH,
+ opt_ssl_crlpath);
+ }
+ mysql_options(connection,MYSQL_OPT_SSL_VERIFY_SERVER_CERT,
+ (char*)&opt_ssl_verify_server_cert);
+#endif
+
+ if (!mysql_real_connect(connection,
+ opt_host ? opt_host : "localhost",
+ opt_user,
+ opt_password,
+ "" /*database*/, opt_port,
+ opt_socket, 0)) {
+ msg("Failed to connect to MySQL server: %s.\n",
+ mysql_error(connection));
+ mysql_close(connection);
+ return(NULL);
+ }
+
+ xb_mysql_query(connection, "SET SESSION wait_timeout=2147483",
+ false, true);
+
+ return(connection);
+}
+
+/*********************************************************************//**
+Execute mysql query. */
+MYSQL_RES *
+xb_mysql_query(MYSQL *connection, const char *query, bool use_result,
+ bool die_on_error)
+{
+ MYSQL_RES *mysql_result = NULL;
+
+ if (mysql_query(connection, query)) {
+ msg("Error: failed to execute query %s: %s\n", query,
+ mysql_error(connection));
+ if (die_on_error) {
+ exit(EXIT_FAILURE);
+ }
+ return(NULL);
+ }
+
+ /* store result set on client if there is a result */
+ if (mysql_field_count(connection) > 0) {
+ if ((mysql_result = mysql_store_result(connection)) == NULL) {
+ msg("Error: failed to fetch query result %s: %s\n",
+ query, mysql_error(connection));
+ exit(EXIT_FAILURE);
+ }
+
+ if (!use_result) {
+ mysql_free_result(mysql_result);
+ }
+ }
+
+ return mysql_result;
+}
+
+
+struct mysql_variable {
+ const char *name;
+ char **value;
+};
+
+
+static
+void
+read_mysql_variables(MYSQL *connection, const char *query, mysql_variable *vars,
+ bool vertical_result)
+{
+ MYSQL_RES *mysql_result;
+ MYSQL_ROW row;
+ mysql_variable *var;
+
+ mysql_result = xb_mysql_query(connection, query, true);
+
+ ut_ad(!vertical_result || mysql_num_fields(mysql_result) == 2);
+
+ if (vertical_result) {
+ while ((row = mysql_fetch_row(mysql_result))) {
+ char *name = row[0];
+ char *value = row[1];
+ for (var = vars; var->name; var++) {
+ if (strcmp(var->name, name) == 0
+ && value != NULL) {
+ *(var->value) = strdup(value);
+ }
+ }
+ }
+ } else {
+ MYSQL_FIELD *field;
+
+ if ((row = mysql_fetch_row(mysql_result)) != NULL) {
+ int i = 0;
+ while ((field = mysql_fetch_field(mysql_result))
+ != NULL) {
+ char *name = field->name;
+ char *value = row[i];
+ for (var = vars; var->name; var++) {
+ if (strcmp(var->name, name) == 0
+ && value != NULL) {
+ *(var->value) = strdup(value);
+ }
+ }
+ ++i;
+ }
+ }
+ }
+
+ mysql_free_result(mysql_result);
+}
+
+
+static
+void
+free_mysql_variables(mysql_variable *vars)
+{
+ mysql_variable *var;
+
+ for (var = vars; var->name; var++) {
+ free(*(var->value));
+ }
+}
+
+
+static
+char *
+read_mysql_one_value(MYSQL *connection, const char *query)
+{
+ MYSQL_RES *mysql_result;
+ MYSQL_ROW row;
+ char *result = NULL;
+
+ mysql_result = xb_mysql_query(connection, query, true);
+
+ ut_ad(mysql_num_fields(mysql_result) == 1);
+
+ if ((row = mysql_fetch_row(mysql_result))) {
+ result = strdup(row[0]);
+ }
+
+ mysql_free_result(mysql_result);
+
+ return(result);
+}
+
+static
+bool
+check_server_version(unsigned long version_number,
+ const char *version_string,
+ const char *version_comment,
+ const char *innodb_version)
+{
+ bool version_supported = false;
+ bool mysql51 = false;
+
+ mysql_server_version = version_number;
+
+ server_flavor = FLAVOR_UNKNOWN;
+ if (strstr(version_comment, "Percona") != NULL) {
+ server_flavor = FLAVOR_PERCONA_SERVER;
+ } else if (strstr(version_comment, "MariaDB") != NULL ||
+ strstr(version_string, "MariaDB") != NULL) {
+ server_flavor = FLAVOR_MARIADB;
+ } else if (strstr(version_comment, "MySQL") != NULL) {
+ server_flavor = FLAVOR_MYSQL;
+ }
+
+ mysql51 = version_number > 50100 && version_number < 50500;
+ version_supported = version_supported
+ || (mysql51 && innodb_version != NULL);
+ version_supported = version_supported
+ || (version_number > 50500 && version_number < 50700);
+ version_supported = version_supported
+ || ((version_number > 100000)
+ && server_flavor == FLAVOR_MARIADB);
+
+ if (mysql51 && innodb_version == NULL) {
+ msg("Error: Built-in InnoDB in MySQL 5.1 is not "
+ "supported in this release. You can either use "
+ "Percona XtraBackup 2.0, or upgrade to InnoDB "
+ "plugin.\n");
+ } else if (!version_supported) {
+ msg("Error: Unsupported server version: '%s'. Please "
+ "report a bug at "
+ "https://bugs.launchpad.net/percona-xtrabackup\n",
+ version_string);
+ }
+
+ return(version_supported);
+}
+
+/*********************************************************************//**
+Receive options important for XtraBackup from MySQL server.
+@return true on success. */
+bool
+get_mysql_vars(MYSQL *connection)
+{
+ char *gtid_mode_var = NULL;
+ char *version_var = NULL;
+ char *version_comment_var = NULL;
+ char *innodb_version_var = NULL;
+ char *have_backup_locks_var = NULL;
+ char *have_backup_safe_binlog_info_var = NULL;
+ char *log_bin_var = NULL;
+ char *lock_wait_timeout_var= NULL;
+ char *wsrep_on_var = NULL;
+ char *slave_parallel_workers_var = NULL;
+ char *gtid_slave_pos_var = NULL;
+ char *innodb_buffer_pool_filename_var = NULL;
+ char *datadir_var = NULL;
+ char *innodb_log_group_home_dir_var = NULL;
+ char *innodb_log_file_size_var = NULL;
+ char *innodb_log_files_in_group_var = NULL;
+ char *innodb_data_file_path_var = NULL;
+ char *innodb_data_home_dir_var = NULL;
+ char *innodb_undo_directory_var = NULL;
+ char *innodb_page_size_var = NULL;
+ char *innodb_undo_tablespaces_var = NULL;
+ char *endptr;
+ unsigned long server_version = mysql_get_server_version(connection);
+
+ bool ret = true;
+
+ mysql_variable mysql_vars[] = {
+ {"have_backup_locks", &have_backup_locks_var},
+ {"have_backup_safe_binlog_info",
+ &have_backup_safe_binlog_info_var},
+ {"log_bin", &log_bin_var},
+ {"lock_wait_timeout", &lock_wait_timeout_var},
+ {"gtid_mode", >id_mode_var},
+ {"version", &version_var},
+ {"version_comment", &version_comment_var},
+ {"innodb_version", &innodb_version_var},
+ {"wsrep_on", &wsrep_on_var},
+ {"slave_parallel_workers", &slave_parallel_workers_var},
+ {"gtid_slave_pos", >id_slave_pos_var},
+ {"innodb_buffer_pool_filename",
+ &innodb_buffer_pool_filename_var},
+ {"datadir", &datadir_var},
+ {"innodb_log_group_home_dir", &innodb_log_group_home_dir_var},
+ {"innodb_log_file_size", &innodb_log_file_size_var},
+ {"innodb_log_files_in_group", &innodb_log_files_in_group_var},
+ {"innodb_data_file_path", &innodb_data_file_path_var},
+ {"innodb_data_home_dir", &innodb_data_home_dir_var},
+ {"innodb_undo_directory", &innodb_undo_directory_var},
+ {"innodb_page_size", &innodb_page_size_var},
+ {"innodb_undo_tablespaces", &innodb_undo_tablespaces_var},
+ {NULL, NULL}
+ };
+
+ read_mysql_variables(connection, "SHOW VARIABLES",
+ mysql_vars, true);
+
+ if (have_backup_locks_var != NULL && !opt_no_backup_locks) {
+ have_backup_locks = true;
+ }
+
+ if (opt_binlog_info == BINLOG_INFO_AUTO) {
+
+ if (have_backup_safe_binlog_info_var != NULL)
+ opt_binlog_info = BINLOG_INFO_LOCKLESS;
+ else if (log_bin_var != NULL && !strcmp(log_bin_var, "ON"))
+ opt_binlog_info = BINLOG_INFO_ON;
+ else
+ opt_binlog_info = BINLOG_INFO_OFF;
+ }
+
+ if (have_backup_safe_binlog_info_var == NULL &&
+ opt_binlog_info == BINLOG_INFO_LOCKLESS) {
+
+ msg("Error: --binlog-info=LOCKLESS is not supported by the "
+ "server\n");
+ return(false);
+ }
+
+ if (lock_wait_timeout_var != NULL) {
+ have_lock_wait_timeout = true;
+ }
+
+ if (wsrep_on_var != NULL) {
+ have_galera_enabled = true;
+ }
+
+ /* Check server version compatibility and detect server flavor */
+
+ if (!(ret = check_server_version(server_version, version_var,
+ version_comment_var,
+ innodb_version_var))) {
+ goto out;
+ }
+
+ if (server_version > 50500) {
+ have_flush_engine_logs = true;
+ }
+
+ if (slave_parallel_workers_var != NULL
+ && atoi(slave_parallel_workers_var) > 0) {
+ have_multi_threaded_slave = true;
+ }
+
+ if (innodb_buffer_pool_filename_var != NULL) {
+ buffer_pool_filename = strdup(innodb_buffer_pool_filename_var);
+ }
+
+ if ((gtid_mode_var && strcmp(gtid_mode_var, "ON") == 0) ||
+ (gtid_slave_pos_var && *gtid_slave_pos_var)) {
+ have_gtid_slave = true;
+ }
+
+ msg("Using server version %s\n", version_var);
+
+ if (!(ret = detect_mysql_capabilities_for_backup())) {
+ goto out;
+ }
+
+ /* make sure datadir value is the same in configuration file */
+ if (check_if_param_set("datadir")) {
+ if (!directory_exists(mysql_data_home, false)) {
+ msg("Warning: option 'datadir' points to "
+ "nonexistent directory '%s'\n", mysql_data_home);
+ }
+ if (!directory_exists(datadir_var, false)) {
+ msg("Warning: MySQL variable 'datadir' points to "
+ "nonexistent directory '%s'\n", datadir_var);
+ }
+ if (!equal_paths(mysql_data_home, datadir_var)) {
+ msg("Warning: option 'datadir' has different "
+ "values:\n"
+ " '%s' in defaults file\n"
+ " '%s' in SHOW VARIABLES\n",
+ mysql_data_home, datadir_var);
+ }
+ }
+
+ /* get some default values is they are missing from my.cnf */
+ if (datadir_var && *datadir_var) {
+ strmake(mysql_real_data_home, datadir_var, FN_REFLEN - 1);
+ mysql_data_home= mysql_real_data_home;
+ }
+
+ if (innodb_data_file_path_var && *innodb_data_file_path_var) {
+ innobase_data_file_path = my_strdup(
+ innodb_data_file_path_var, MYF(MY_FAE));
+ }
+
+ if (innodb_data_home_dir_var && *innodb_data_home_dir_var) {
+ innobase_data_home_dir = my_strdup(
+ innodb_data_home_dir_var, MYF(MY_FAE));
+ }
+
+ if (innodb_log_group_home_dir_var
+ && *innodb_log_group_home_dir_var) {
+ srv_log_group_home_dir = my_strdup(
+ innodb_log_group_home_dir_var, MYF(MY_FAE));
+ }
+
+ if (innodb_undo_directory_var && *innodb_undo_directory_var) {
+ srv_undo_dir = my_strdup(
+ innodb_undo_directory_var, MYF(MY_FAE));
+ }
+
+ if (innodb_log_files_in_group_var) {
+ srv_n_log_files = strtol(
+ innodb_log_files_in_group_var, &endptr, 10);
+ ut_ad(*endptr == 0);
+ }
+
+ if (innodb_log_file_size_var) {
+ srv_log_file_size = strtoll(
+ innodb_log_file_size_var, &endptr, 10);
+ ut_ad(*endptr == 0);
+ }
+
+ if (innodb_page_size_var) {
+ innobase_page_size = strtoll(
+ innodb_page_size_var, &endptr, 10);
+ ut_ad(*endptr == 0);
+ }
+
+ if (innodb_undo_tablespaces_var) {
+ srv_undo_tablespaces = strtoul(innodb_undo_tablespaces_var, &endptr, 10);
+ ut_ad(*endptr == 0);
+ }
+
+out:
+ free_mysql_variables(mysql_vars);
+
+ return(ret);
+}
+
+/*********************************************************************//**
+Query the server to find out what backup capabilities it supports.
+@return true on success. */
+bool
+detect_mysql_capabilities_for_backup()
+{
+ const char *query = "SELECT 'INNODB_CHANGED_PAGES', COUNT(*) FROM "
+ "INFORMATION_SCHEMA.PLUGINS "
+ "WHERE PLUGIN_NAME LIKE 'INNODB_CHANGED_PAGES'";
+ char *innodb_changed_pages = NULL;
+ mysql_variable vars[] = {
+ {"INNODB_CHANGED_PAGES", &innodb_changed_pages}, {NULL, NULL}};
+
+ if (xtrabackup_incremental) {
+
+ read_mysql_variables(mysql_connection, query, vars, true);
+
+ ut_ad(innodb_changed_pages != NULL);
+
+ have_changed_page_bitmaps = (atoi(innodb_changed_pages) == 1);
+
+ /* INNODB_CHANGED_PAGES are listed in
+ INFORMATION_SCHEMA.PLUGINS in MariaDB, but
+ FLUSH NO_WRITE_TO_BINLOG CHANGED_PAGE_BITMAPS
+ is not supported for versions below 10.1.6
+ (see MDEV-7472) */
+ if (server_flavor == FLAVOR_MARIADB &&
+ mysql_server_version < 100106) {
+ have_changed_page_bitmaps = false;
+ }
+
+ free_mysql_variables(vars);
+ }
+
+ /* do some sanity checks */
+ if (opt_galera_info && !have_galera_enabled) {
+ msg("--galera-info is specified on the command "
+ "line, but the server does not support Galera "
+ "replication. Ignoring the option.\n");
+ opt_galera_info = false;
+ }
+
+ if (opt_slave_info && have_multi_threaded_slave &&
+ !have_gtid_slave) {
+ msg("The --slave-info option requires GTID enabled for a "
+ "multi-threaded slave.\n");
+ return(false);
+ }
+
+ return(true);
+}
+
+static
+bool
+select_incremental_lsn_from_history(lsn_t *incremental_lsn)
+{
+ MYSQL_RES *mysql_result;
+ char query[1000];
+ char buf[100];
+
+ if (opt_incremental_history_name) {
+ mysql_real_escape_string(mysql_connection, buf,
+ opt_incremental_history_name,
+ (unsigned long)strlen(opt_incremental_history_name));
+ ut_snprintf(query, sizeof(query),
+ "SELECT innodb_to_lsn "
+ "FROM PERCONA_SCHEMA.xtrabackup_history "
+ "WHERE name = '%s' "
+ "AND innodb_to_lsn IS NOT NULL "
+ "ORDER BY innodb_to_lsn DESC LIMIT 1",
+ buf);
+ }
+
+ if (opt_incremental_history_uuid) {
+ mysql_real_escape_string(mysql_connection, buf,
+ opt_incremental_history_uuid,
+ (unsigned long)strlen(opt_incremental_history_uuid));
+ ut_snprintf(query, sizeof(query),
+ "SELECT innodb_to_lsn "
+ "FROM PERCONA_SCHEMA.xtrabackup_history "
+ "WHERE uuid = '%s' "
+ "AND innodb_to_lsn IS NOT NULL "
+ "ORDER BY innodb_to_lsn DESC LIMIT 1",
+ buf);
+ }
+
+ mysql_result = xb_mysql_query(mysql_connection, query, true);
+
+ ut_ad(mysql_num_fields(mysql_result) == 1);
+ const MYSQL_ROW row = mysql_fetch_row(mysql_result);
+ if (row) {
+ *incremental_lsn = strtoull(row[0], NULL, 10);
+ msg("Found and using lsn: " LSN_PF " for %s %s\n",
+ *incremental_lsn,
+ opt_incremental_history_uuid ? "uuid" : "name",
+ opt_incremental_history_uuid ?
+ opt_incremental_history_uuid :
+ opt_incremental_history_name);
+ } else {
+ msg("Error while attempting to find history record "
+ "for %s %s\n",
+ opt_incremental_history_uuid ? "uuid" : "name",
+ opt_incremental_history_uuid ?
+ opt_incremental_history_uuid :
+ opt_incremental_history_name);
+ }
+
+ mysql_free_result(mysql_result);
+
+ return(row != NULL);
+}
+
+static
+const char *
+eat_sql_whitespace(const char *query)
+{
+ bool comment = false;
+
+ while (*query) {
+ if (comment) {
+ if (query[0] == '*' && query[1] == '/') {
+ query += 2;
+ comment = false;
+ continue;
+ }
+ ++query;
+ continue;
+ }
+ if (query[0] == '/' && query[1] == '*') {
+ query += 2;
+ comment = true;
+ continue;
+ }
+ if (strchr("\t\n\r (", query[0])) {
+ ++query;
+ continue;
+ }
+ break;
+ }
+
+ return(query);
+}
+
+static
+bool
+is_query_from_list(const char *query, const char **list)
+{
+ const char **item;
+
+ query = eat_sql_whitespace(query);
+
+ item = list;
+ while (*item) {
+ if (strncasecmp(query, *item, strlen(*item)) == 0) {
+ return(true);
+ }
+ ++item;
+ }
+
+ return(false);
+}
+
+static
+bool
+is_query(const char *query)
+{
+ const char *query_list[] = {"insert", "update", "delete", "replace",
+ "alter", "load", "select", "do", "handler", "call", "execute",
+ "begin", NULL};
+
+ return is_query_from_list(query, query_list);
+}
+
+static
+bool
+is_select_query(const char *query)
+{
+ const char *query_list[] = {"select", NULL};
+
+ return is_query_from_list(query, query_list);
+}
+
+static
+bool
+is_update_query(const char *query)
+{
+ const char *query_list[] = {"insert", "update", "delete", "replace",
+ "alter", "load", NULL};
+
+ return is_query_from_list(query, query_list);
+}
+
+static
+bool
+have_queries_to_wait_for(MYSQL *connection, uint threshold)
+{
+ MYSQL_RES *result = xb_mysql_query(connection, "SHOW FULL PROCESSLIST",
+ true);
+ const bool all_queries = (opt_lock_wait_query_type == QUERY_TYPE_ALL);
+ bool have_to_wait = false;
+
+ while (MYSQL_ROW row = mysql_fetch_row(result)) {
+ const char *info = row[7];
+ int duration = row[5] ? atoi(row[5]) : 0;
+ char *id = row[0];
+
+ if (info != NULL
+ && duration >= (int)threshold
+ && ((all_queries && is_query(info))
+ || is_update_query(info))) {
+ msg_ts("Waiting for query %s (duration %d sec): %s",
+ id, duration, info);
+ have_to_wait = true;
+ break;
+ }
+ }
+
+ mysql_free_result(result);
+ return(have_to_wait);
+}
+
+static
+void
+kill_long_queries(MYSQL *connection, time_t timeout)
+{
+ char kill_stmt[100];
+
+ MYSQL_RES *result = xb_mysql_query(connection, "SHOW FULL PROCESSLIST",
+ true);
+ const bool all_queries = (opt_kill_long_query_type == QUERY_TYPE_ALL);
+ while (MYSQL_ROW row = mysql_fetch_row(result)) {
+ const char *info = row[7];
+ long long duration = row[5]? atoll(row[5]) : 0;
+ char *id = row[0];
+
+ if (info != NULL &&
+ (time_t)duration >= timeout &&
+ ((all_queries && is_query(info)) ||
+ is_select_query(info))) {
+ msg_ts("Killing query %s (duration %d sec): %s\n",
+ id, (int)duration, info);
+ ut_snprintf(kill_stmt, sizeof(kill_stmt),
+ "KILL %s", id);
+ xb_mysql_query(connection, kill_stmt, false, false);
+ }
+ }
+
+ mysql_free_result(result);
+}
+
+static
+bool
+wait_for_no_updates(MYSQL *connection, uint timeout, uint threshold)
+{
+ time_t start_time;
+
+ start_time = time(NULL);
+
+ msg_ts("Waiting %u seconds for queries running longer than %u seconds "
+ "to finish\n", timeout, threshold);
+
+ while (time(NULL) <= (time_t)(start_time + timeout)) {
+ if (!have_queries_to_wait_for(connection, threshold)) {
+ return(true);
+ }
+ os_thread_sleep(1000000);
+ }
+
+ msg_ts("Unable to obtain lock. Please try again later.");
+
+ return(false);
+}
+
+static
+os_thread_ret_t
+kill_query_thread(
+/*===============*/
+ void *arg __attribute__((unused)))
+{
+ MYSQL *mysql;
+ time_t start_time;
+
+ start_time = time(NULL);
+
+ os_event_set(kill_query_thread_started);
+
+ msg_ts("Kill query timeout %d seconds.\n",
+ opt_kill_long_queries_timeout);
+
+ while (time(NULL) - start_time <
+ (time_t)opt_kill_long_queries_timeout) {
+ if (os_event_wait_time(kill_query_thread_stop, 1000) !=
+ OS_SYNC_TIME_EXCEEDED) {
+ goto stop_thread;
+ }
+ }
+
+ if ((mysql = xb_mysql_connect()) == NULL) {
+ msg("Error: kill query thread failed\n");
+ goto stop_thread;
+ }
+
+ while (true) {
+ kill_long_queries(mysql, time(NULL) - start_time);
+ if (os_event_wait_time(kill_query_thread_stop, 1000) !=
+ OS_SYNC_TIME_EXCEEDED) {
+ break;
+ }
+ }
+
+ mysql_close(mysql);
+
+stop_thread:
+ msg_ts("Kill query thread stopped\n");
+
+ os_event_set(kill_query_thread_stopped);
+
+ os_thread_exit();
+ OS_THREAD_DUMMY_RETURN;
+}
+
+
+static
+void
+start_query_killer()
+{
+ kill_query_thread_stop = os_event_create(0);
+ kill_query_thread_started = os_event_create(0);
+ kill_query_thread_stopped = os_event_create(0);
+
+ os_thread_create(kill_query_thread, NULL, &kill_query_thread_id);
+
+ os_event_wait(kill_query_thread_started);
+}
+
+static
+void
+stop_query_killer()
+{
+ os_event_set(kill_query_thread_stop);
+ os_event_wait_time(kill_query_thread_stopped, 60000);
+}
+
+/*********************************************************************//**
+Function acquires either a backup tables lock, if supported
+by the server, or a global read lock (FLUSH TABLES WITH READ LOCK)
+otherwise.
+@returns true if lock acquired */
+bool
+lock_tables(MYSQL *connection)
+{
+ if (have_lock_wait_timeout) {
+ /* Set the maximum supported session value for
+ lock_wait_timeout to prevent unnecessary timeouts when the
+ global value is changed from the default */
+ xb_mysql_query(connection,
+ "SET SESSION lock_wait_timeout=31536000", false);
+ }
+
+ if (have_backup_locks) {
+ msg_ts("Executing LOCK TABLES FOR BACKUP...\n");
+ xb_mysql_query(connection, "LOCK TABLES FOR BACKUP", false);
+ return(true);
+ }
+
+ if (!opt_lock_wait_timeout && !opt_kill_long_queries_timeout) {
+
+ /* We do first a FLUSH TABLES. If a long update is running, the
+ FLUSH TABLES will wait but will not stall the whole mysqld, and
+ when the long update is done the FLUSH TABLES WITH READ LOCK
+ will start and succeed quickly. So, FLUSH TABLES is to lower
+ the probability of a stage where both mysqldump and most client
+ connections are stalled. Of course, if a second long update
+ starts between the two FLUSHes, we have that bad stall.
+
+ Option lock_wait_timeout serve the same purpose and is not
+ compatible with this trick.
+ */
+
+ msg_ts("Executing FLUSH NO_WRITE_TO_BINLOG TABLES...\n");
+
+ xb_mysql_query(connection,
+ "FLUSH NO_WRITE_TO_BINLOG TABLES", false);
+ }
+
+ if (opt_lock_wait_timeout) {
+ if (!wait_for_no_updates(connection, opt_lock_wait_timeout,
+ opt_lock_wait_threshold)) {
+ return(false);
+ }
+ }
+
+ msg_ts("Executing FLUSH TABLES WITH READ LOCK...\n");
+
+ if (opt_kill_long_queries_timeout) {
+ start_query_killer();
+ }
+
+ if (have_galera_enabled) {
+ xb_mysql_query(connection,
+ "SET SESSION wsrep_causal_reads=0", false);
+ }
+
+ xb_mysql_query(connection, "FLUSH TABLES WITH READ LOCK", false);
+
+ if (opt_kill_long_queries_timeout) {
+ stop_query_killer();
+ }
+
+ return(true);
+}
+
+
+/*********************************************************************//**
+If backup locks are used, execute LOCK BINLOG FOR BACKUP provided that we are
+not in the --no-lock mode and the lock has not been acquired already.
+@returns true if lock acquired */
+bool
+lock_binlog_maybe(MYSQL *connection)
+{
+ if (have_backup_locks && !opt_no_lock && !binlog_locked) {
+ msg_ts("Executing LOCK BINLOG FOR BACKUP...\n");
+ xb_mysql_query(connection, "LOCK BINLOG FOR BACKUP", false);
+ binlog_locked = true;
+
+ return(true);
+ }
+
+ return(false);
+}
+
+
+/*********************************************************************//**
+Releases either global read lock acquired with FTWRL and the binlog
+lock acquired with LOCK BINLOG FOR BACKUP, depending on
+the locking strategy being used */
+void
+unlock_all(MYSQL *connection)
+{
+ if (opt_debug_sleep_before_unlock) {
+ msg_ts("Debug sleep for %u seconds\n",
+ opt_debug_sleep_before_unlock);
+ os_thread_sleep(opt_debug_sleep_before_unlock * 1000);
+ }
+
+ if (binlog_locked) {
+ msg_ts("Executing UNLOCK BINLOG\n");
+ xb_mysql_query(connection, "UNLOCK BINLOG", false);
+ }
+
+ msg_ts("Executing UNLOCK TABLES\n");
+ xb_mysql_query(connection, "UNLOCK TABLES", false);
+
+ msg_ts("All tables unlocked\n");
+}
+
+
+static
+int
+get_open_temp_tables(MYSQL *connection)
+{
+ char *slave_open_temp_tables = NULL;
+ mysql_variable status[] = {
+ {"Slave_open_temp_tables", &slave_open_temp_tables},
+ {NULL, NULL}
+ };
+ int result = false;
+
+ read_mysql_variables(connection,
+ "SHOW STATUS LIKE 'slave_open_temp_tables'", status, true);
+
+ result = slave_open_temp_tables ? atoi(slave_open_temp_tables) : 0;
+
+ free_mysql_variables(status);
+
+ return(result);
+}
+
+/*********************************************************************//**
+Wait until it's safe to backup a slave. Returns immediately if
+the host isn't a slave. Currently there's only one check:
+Slave_open_temp_tables has to be zero. Dies on timeout. */
+bool
+wait_for_safe_slave(MYSQL *connection)
+{
+ char *read_master_log_pos = NULL;
+ char *slave_sql_running = NULL;
+ int n_attempts = 1;
+ const int sleep_time = 3;
+ int open_temp_tables = 0;
+ bool result = true;
+
+ mysql_variable status[] = {
+ {"Read_Master_Log_Pos", &read_master_log_pos},
+ {"Slave_SQL_Running", &slave_sql_running},
+ {NULL, NULL}
+ };
+
+ sql_thread_started = false;
+
+ read_mysql_variables(connection, "SHOW SLAVE STATUS", status, false);
+
+ if (!(read_master_log_pos && slave_sql_running)) {
+ msg("Not checking slave open temp tables for "
+ "--safe-slave-backup because host is not a slave\n");
+ goto cleanup;
+ }
+
+ if (strcmp(slave_sql_running, "Yes") == 0) {
+ sql_thread_started = true;
+ xb_mysql_query(connection, "STOP SLAVE SQL_THREAD", false);
+ }
+
+ if (opt_safe_slave_backup_timeout > 0) {
+ n_attempts = opt_safe_slave_backup_timeout / sleep_time;
+ }
+
+ open_temp_tables = get_open_temp_tables(connection);
+ msg_ts("Slave open temp tables: %d\n", open_temp_tables);
+
+ while (open_temp_tables && n_attempts--) {
+ msg_ts("Starting slave SQL thread, waiting %d seconds, then "
+ "checking Slave_open_temp_tables again (%d attempts "
+ "remaining)...\n", sleep_time, n_attempts);
+
+ xb_mysql_query(connection, "START SLAVE SQL_THREAD", false);
+ os_thread_sleep(sleep_time * 1000000);
+ xb_mysql_query(connection, "STOP SLAVE SQL_THREAD", false);
+
+ open_temp_tables = get_open_temp_tables(connection);
+ msg_ts("Slave open temp tables: %d\n", open_temp_tables);
+ }
+
+ /* Restart the slave if it was running at start */
+ if (open_temp_tables == 0) {
+ msg_ts("Slave is safe to backup\n");
+ goto cleanup;
+ }
+
+ result = false;
+
+ if (sql_thread_started) {
+ msg_ts("Restarting slave SQL thread.\n");
+ xb_mysql_query(connection, "START SLAVE SQL_THREAD", false);
+ }
+
+ msg_ts("Slave_open_temp_tables did not become zero after "
+ "%d seconds\n", opt_safe_slave_backup_timeout);
+
+cleanup:
+ free_mysql_variables(status);
+
+ return(result);
+}
+
+
+/*********************************************************************//**
+Retrieves MySQL binlog position of the master server in a replication
+setup and saves it in a file. It also saves it in mysql_slave_position
+variable. */
+bool
+write_slave_info(MYSQL *connection)
+{
+ char *master = NULL;
+ char *filename = NULL;
+ char *gtid_executed = NULL;
+ char *position = NULL;
+ char *gtid_slave_pos = NULL;
+ char *ptr;
+ bool result = false;
+
+ mysql_variable status[] = {
+ {"Master_Host", &master},
+ {"Relay_Master_Log_File", &filename},
+ {"Exec_Master_Log_Pos", &position},
+ {"Executed_Gtid_Set", >id_executed},
+ {NULL, NULL}
+ };
+
+ mysql_variable variables[] = {
+ {"gtid_slave_pos", >id_slave_pos},
+ {NULL, NULL}
+ };
+
+ read_mysql_variables(connection, "SHOW SLAVE STATUS", status, false);
+ read_mysql_variables(connection, "SHOW VARIABLES", variables, true);
+
+ if (master == NULL || filename == NULL || position == NULL) {
+ msg("Failed to get master binlog coordinates "
+ "from SHOW SLAVE STATUS\n");
+ msg("This means that the server is not a "
+ "replication slave. Ignoring the --slave-info "
+ "option\n");
+ /* we still want to continue the backup */
+ result = true;
+ goto cleanup;
+ }
+
+ /* Print slave status to a file.
+ If GTID mode is used, construct a CHANGE MASTER statement with
+ MASTER_AUTO_POSITION and correct a gtid_purged value. */
+ if (gtid_executed != NULL && *gtid_executed) {
+ /* MySQL >= 5.6 with GTID enabled */
+
+ for (ptr = strchr(gtid_executed, '\n');
+ ptr;
+ ptr = strchr(ptr, '\n')) {
+ *ptr = ' ';
+ }
+
+ result = backup_file_printf(XTRABACKUP_SLAVE_INFO,
+ "SET GLOBAL gtid_purged='%s';\n"
+ "CHANGE MASTER TO MASTER_AUTO_POSITION=1\n",
+ gtid_executed);
+
+ ut_a(asprintf(&mysql_slave_position,
+ "master host '%s', purge list '%s'",
+ master, gtid_executed) != -1);
+ } else if (gtid_slave_pos && *gtid_slave_pos) {
+ /* MariaDB >= 10.0 with GTID enabled */
+ result = backup_file_printf(XTRABACKUP_SLAVE_INFO,
+ "SET GLOBAL gtid_slave_pos = '%s';\n"
+ "CHANGE MASTER TO master_use_gtid = slave_pos\n",
+ gtid_slave_pos);
+ ut_a(asprintf(&mysql_slave_position,
+ "master host '%s', gtid_slave_pos %s",
+ master, gtid_slave_pos) != -1);
+ } else {
+ result = backup_file_printf(XTRABACKUP_SLAVE_INFO,
+ "CHANGE MASTER TO MASTER_LOG_FILE='%s', "
+ "MASTER_LOG_POS=%s\n", filename, position);
+ ut_a(asprintf(&mysql_slave_position,
+ "master host '%s', filename '%s', position '%s'",
+ master, filename, position) != -1);
+ }
+
+cleanup:
+ free_mysql_variables(status);
+ free_mysql_variables(variables);
+
+ return(result);
+}
+
+
+/*********************************************************************//**
+Retrieves MySQL Galera and
+saves it in a file. It also prints it to stdout. */
+bool
+write_galera_info(MYSQL *connection)
+{
+ char *state_uuid = NULL, *state_uuid55 = NULL;
+ char *last_committed = NULL, *last_committed55 = NULL;
+ bool result;
+
+ mysql_variable status[] = {
+ {"Wsrep_local_state_uuid", &state_uuid},
+ {"wsrep_local_state_uuid", &state_uuid55},
+ {"Wsrep_last_committed", &last_committed},
+ {"wsrep_last_committed", &last_committed55},
+ {NULL, NULL}
+ };
+
+ /* When backup locks are supported by the server, we should skip
+ creating xtrabackup_galera_info file on the backup stage, because
+ wsrep_local_state_uuid and wsrep_last_committed will be inconsistent
+ without blocking commits. The state file will be created on the prepare
+ stage using the WSREP recovery procedure. */
+ if (have_backup_locks) {
+ return(true);
+ }
+
+ read_mysql_variables(connection, "SHOW STATUS", status, true);
+
+ if ((state_uuid == NULL && state_uuid55 == NULL)
+ || (last_committed == NULL && last_committed55 == NULL)) {
+ msg("Failed to get master wsrep state from SHOW STATUS.\n");
+ result = false;
+ goto cleanup;
+ }
+
+ result = backup_file_printf(XTRABACKUP_GALERA_INFO,
+ "%s:%s\n", state_uuid ? state_uuid : state_uuid55,
+ last_committed ? last_committed : last_committed55);
+
+cleanup:
+ free_mysql_variables(status);
+
+ return(result);
+}
+
+
+/*********************************************************************//**
+Flush and copy the current binary log file into the backup,
+if GTID is enabled */
+bool
+write_current_binlog_file(MYSQL *connection)
+{
+ char *executed_gtid_set = NULL;
+ char *gtid_binlog_state = NULL;
+ char *log_bin_file = NULL;
+ char *log_bin_dir = NULL;
+ bool gtid_exists;
+ bool result = true;
+ char filepath[FN_REFLEN];
+
+ mysql_variable status[] = {
+ {"Executed_Gtid_Set", &executed_gtid_set},
+ {NULL, NULL}
+ };
+
+ mysql_variable status_after_flush[] = {
+ {"File", &log_bin_file},
+ {NULL, NULL}
+ };
+
+ mysql_variable vars[] = {
+ {"gtid_binlog_state", >id_binlog_state},
+ {"log_bin_basename", &log_bin_dir},
+ {NULL, NULL}
+ };
+
+ read_mysql_variables(connection, "SHOW MASTER STATUS", status, false);
+ read_mysql_variables(connection, "SHOW VARIABLES", vars, true);
+
+ gtid_exists = (executed_gtid_set && *executed_gtid_set)
+ || (gtid_binlog_state && *gtid_binlog_state);
+
+ if (gtid_exists) {
+ size_t log_bin_dir_length;
+
+ lock_binlog_maybe(connection);
+
+ xb_mysql_query(connection, "FLUSH BINARY LOGS", false);
+
+ read_mysql_variables(connection, "SHOW MASTER STATUS",
+ status_after_flush, false);
+
+ if (opt_log_bin != NULL && strchr(opt_log_bin, FN_LIBCHAR)) {
+ /* If log_bin is set, it has priority */
+ if (log_bin_dir) {
+ free(log_bin_dir);
+ }
+ log_bin_dir = strdup(opt_log_bin);
+ } else if (log_bin_dir == NULL) {
+ /* Default location is MySQL datadir */
+ log_bin_dir = strdup("./");
+ }
+
+ dirname_part(log_bin_dir, log_bin_dir, &log_bin_dir_length);
+
+ /* strip final slash if it is not the only path component */
+ if (log_bin_dir_length > 1 &&
+ log_bin_dir[log_bin_dir_length - 1] == FN_LIBCHAR) {
+ log_bin_dir[log_bin_dir_length - 1] = 0;
+ }
+
+ if (log_bin_dir == NULL || log_bin_file == NULL) {
+ msg("Failed to get master binlog coordinates from "
+ "SHOW MASTER STATUS");
+ result = false;
+ goto cleanup;
+ }
+
+ ut_snprintf(filepath, sizeof(filepath), "%s%c%s",
+ log_bin_dir, FN_LIBCHAR, log_bin_file);
+ result = copy_file(ds_data, filepath, log_bin_file, 0);
+ }
+
+cleanup:
+ free_mysql_variables(status_after_flush);
+ free_mysql_variables(status);
+ free_mysql_variables(vars);
+
+ return(result);
+}
+
+
+/*********************************************************************//**
+Retrieves MySQL binlog position and
+saves it in a file. It also prints it to stdout. */
+bool
+write_binlog_info(MYSQL *connection)
+{
+ char *filename = NULL;
+ char *position = NULL;
+ char *gtid_mode = NULL;
+ char *gtid_current_pos = NULL;
+ char *gtid_executed = NULL;
+ char *gtid = NULL;
+ bool result;
+ bool mysql_gtid;
+ bool mariadb_gtid;
+
+ mysql_variable status[] = {
+ {"File", &filename},
+ {"Position", &position},
+ {"Executed_Gtid_Set", >id_executed},
+ {NULL, NULL}
+ };
+
+ mysql_variable vars[] = {
+ {"gtid_mode", >id_mode},
+ {"gtid_current_pos", >id_current_pos},
+ {NULL, NULL}
+ };
+
+ read_mysql_variables(connection, "SHOW MASTER STATUS", status, false);
+ read_mysql_variables(connection, "SHOW VARIABLES", vars, true);
+
+ if (filename == NULL || position == NULL) {
+ /* Do not create xtrabackup_binlog_info if binary
+ log is disabled */
+ result = true;
+ goto cleanup;
+ }
+
+ mysql_gtid = ((gtid_mode != NULL) && (strcmp(gtid_mode, "ON") == 0));
+ mariadb_gtid = (gtid_current_pos != NULL);
+
+ gtid = (gtid_executed != NULL ? gtid_executed : gtid_current_pos);
+
+ if (mariadb_gtid || mysql_gtid) {
+ ut_a(asprintf(&mysql_binlog_position,
+ "filename '%s', position '%s', "
+ "GTID of the last change '%s'",
+ filename, position, gtid) != -1);
+ result = backup_file_printf(XTRABACKUP_BINLOG_INFO,
+ "%s\t%s\t%s\n", filename, position,
+ gtid);
+ } else {
+ ut_a(asprintf(&mysql_binlog_position,
+ "filename '%s', position '%s'",
+ filename, position) != -1);
+ result = backup_file_printf(XTRABACKUP_BINLOG_INFO,
+ "%s\t%s\n", filename, position);
+ }
+
+cleanup:
+ free_mysql_variables(status);
+ free_mysql_variables(vars);
+
+ return(result);
+}
+
+struct escape_and_quote
+{
+ escape_and_quote(MYSQL *mysql, const char *str)
+ : mysql(mysql), str(str) {}
+ MYSQL * const mysql;
+ const char * const str;
+};
+
+static
+std::ostream&
+operator<<(std::ostream& s, const escape_and_quote& eq)
+{
+ if (!eq.str)
+ return s << "NULL";
+ s << '\'';
+ size_t len = strlen(eq.str);
+ char* escaped = (char *)alloca(2 * len + 1);
+ len = mysql_real_escape_string(eq.mysql, escaped, eq.str, len);
+ s << std::string(escaped, len);
+ s << '\'';
+ return s;
+}
+
+/*********************************************************************//**
+Writes xtrabackup_info file and if backup_history is enable creates
+PERCONA_SCHEMA.xtrabackup_history and writes a new history record to the
+table containing all the history info particular to the just completed
+backup. */
+bool
+write_xtrabackup_info(MYSQL *connection)
+{
+
+ char *uuid = NULL;
+ char *server_version = NULL;
+ char buf_start_time[100];
+ char buf_end_time[100];
+ tm tm;
+ std::ostringstream oss;
+ const char *xb_stream_name[] = {"file", "tar", "xbstream"};
+
+ uuid = read_mysql_one_value(connection, "SELECT UUID()");
+ server_version = read_mysql_one_value(connection, "SELECT VERSION()");
+ localtime_r(&history_start_time, &tm);
+ strftime(buf_start_time, sizeof(buf_start_time),
+ "%Y-%m-%d %H:%M:%S", &tm);
+ history_end_time = time(NULL);
+ localtime_r(&history_end_time, &tm);
+ strftime(buf_end_time, sizeof(buf_end_time),
+ "%Y-%m-%d %H:%M:%S", &tm);
+ bool is_partial = (xtrabackup_tables
+ || xtrabackup_tables_file
+ || xtrabackup_databases
+ || xtrabackup_databases_file
+ || xtrabackup_tables_exclude
+ || xtrabackup_databases_exclude
+ );
+
+ backup_file_printf(XTRABACKUP_INFO,
+ "uuid = %s\n"
+ "name = %s\n"
+ "tool_name = %s\n"
+ "tool_command = %s\n"
+ "tool_version = %s\n"
+ "ibbackup_version = %s\n"
+ "server_version = %s\n"
+ "start_time = %s\n"
+ "end_time = %s\n"
+ "lock_time = %d\n"
+ "binlog_pos = %s\n"
+ "innodb_from_lsn = %llu\n"
+ "innodb_to_lsn = %llu\n"
+ "partial = %s\n"
+ "incremental = %s\n"
+ "format = %s\n"
+ "compressed = %s\n",
+ uuid, /* uuid */
+ opt_history ? opt_history : "", /* name */
+ tool_name, /* tool_name */
+ tool_args, /* tool_command */
+ MYSQL_SERVER_VERSION, /* tool_version */
+ MYSQL_SERVER_VERSION, /* ibbackup_version */
+ server_version, /* server_version */
+ buf_start_time, /* start_time */
+ buf_end_time, /* end_time */
+ (int)history_lock_time, /* lock_time */
+ mysql_binlog_position ?
+ mysql_binlog_position : "", /* binlog_pos */
+ incremental_lsn, /* innodb_from_lsn */
+ metadata_to_lsn, /* innodb_to_lsn */
+ is_partial? "Y" : "N",
+ xtrabackup_incremental ? "Y" : "N", /* incremental */
+ xb_stream_name[xtrabackup_stream_fmt], /* format */
+ xtrabackup_compress ? "compressed" : "N"); /* compressed */
+
+ if (!opt_history) {
+ goto cleanup;
+ }
+
+ xb_mysql_query(connection,
+ "CREATE DATABASE IF NOT EXISTS PERCONA_SCHEMA", false);
+ xb_mysql_query(connection,
+ "CREATE TABLE IF NOT EXISTS PERCONA_SCHEMA.xtrabackup_history("
+ "uuid VARCHAR(40) NOT NULL PRIMARY KEY,"
+ "name VARCHAR(255) DEFAULT NULL,"
+ "tool_name VARCHAR(255) DEFAULT NULL,"
+ "tool_command TEXT DEFAULT NULL,"
+ "tool_version VARCHAR(255) DEFAULT NULL,"
+ "ibbackup_version VARCHAR(255) DEFAULT NULL,"
+ "server_version VARCHAR(255) DEFAULT NULL,"
+ "start_time TIMESTAMP NULL DEFAULT NULL,"
+ "end_time TIMESTAMP NULL DEFAULT NULL,"
+ "lock_time BIGINT UNSIGNED DEFAULT NULL,"
+ "binlog_pos VARCHAR(128) DEFAULT NULL,"
+ "innodb_from_lsn BIGINT UNSIGNED DEFAULT NULL,"
+ "innodb_to_lsn BIGINT UNSIGNED DEFAULT NULL,"
+ "partial ENUM('Y', 'N') DEFAULT NULL,"
+ "incremental ENUM('Y', 'N') DEFAULT NULL,"
+ "format ENUM('file', 'tar', 'xbstream') DEFAULT NULL,"
+ "compressed ENUM('Y', 'N') DEFAULT NULL"
+ ") CHARACTER SET utf8 ENGINE=innodb", false);
+
+
+#define ESCAPE_BOOL(expr) ((expr)?"'Y'":"'N'")
+
+ oss << "insert into PERCONA_SCHEMA.xtrabackup_history("
+ << "uuid, name, tool_name, tool_command, tool_version,"
+ << "ibbackup_version, server_version, start_time, end_time,"
+ << "lock_time, binlog_pos, innodb_from_lsn, innodb_to_lsn,"
+ << "partial, incremental, format, compressed) "
+ << "values("
+ << escape_and_quote(connection, uuid) << ","
+ << escape_and_quote(connection, opt_history) << ","
+ << escape_and_quote(connection, tool_name) << ","
+ << escape_and_quote(connection, tool_args) << ","
+ << escape_and_quote(connection, MYSQL_SERVER_VERSION) << ","
+ << escape_and_quote(connection, MYSQL_SERVER_VERSION) << ","
+ << escape_and_quote(connection, server_version) << ","
+ << "from_unixtime(" << history_start_time << "),"
+ << "from_unixtime(" << history_end_time << "),"
+ << history_lock_time << ","
+ << escape_and_quote(connection, mysql_binlog_position) << ","
+ << incremental_lsn << ","
+ << metadata_to_lsn << ","
+ << ESCAPE_BOOL(is_partial) << ","
+ << ESCAPE_BOOL(xtrabackup_incremental)<< ","
+ << escape_and_quote(connection,xb_stream_name[xtrabackup_stream_fmt]) <<","
+ << ESCAPE_BOOL(xtrabackup_compress) << ")";
+
+ xb_mysql_query(mysql_connection, oss.str().c_str(), false);
+
+cleanup:
+
+ free(uuid);
+ free(server_version);
+
+ return(true);
+}
+
+extern const char *innodb_checksum_algorithm_names[];
+
+bool write_backup_config_file()
+{
+ int rc= backup_file_printf("backup-my.cnf",
+ "# This MySQL options file was generated by innobackupex.\n\n"
+ "# The MySQL server\n"
+ "[mysqld]\n"
+ "innodb_checksum_algorithm=%s\n"
+ "innodb_data_file_path=%s\n"
+ "innodb_log_files_in_group=%lu\n"
+ "innodb_log_file_size=%llu\n"
+ "innodb_page_size=%lu\n"
+ "innodb_undo_directory=%s\n"
+ "innodb_undo_tablespaces=%lu\n"
+ "%s%s\n"
+ "%s\n",
+ innodb_checksum_algorithm_names[srv_checksum_algorithm],
+ innobase_data_file_path,
+ srv_n_log_files,
+ srv_log_file_size,
+ srv_page_size,
+ srv_undo_dir,
+ srv_undo_tablespaces,
+ innobase_buffer_pool_filename ?
+ "innodb_buffer_pool_filename=" : "",
+ innobase_buffer_pool_filename ?
+ innobase_buffer_pool_filename : "",
+ encryption_plugin_get_config());
+ return rc;
+}
+
+
+static
+char *make_argv(char *buf, size_t len, int argc, char **argv)
+{
+ size_t left= len;
+ const char *arg;
+
+ buf[0]= 0;
+ ++argv; --argc;
+ while (argc > 0 && left > 0)
+ {
+ arg = *argv;
+ if (strncmp(*argv, "--password", strlen("--password")) == 0) {
+ arg = "--password=...";
+ }
+ left-= ut_snprintf(buf + len - left, left,
+ "%s%c", arg, argc > 1 ? ' ' : 0);
+ ++argv; --argc;
+ }
+
+ return buf;
+}
+
+void
+capture_tool_command(int argc, char **argv)
+{
+ /* capture tool name tool args */
+ tool_name = strrchr(argv[0], '/');
+ tool_name = tool_name ? tool_name + 1 : argv[0];
+
+ make_argv(tool_args, sizeof(tool_args), argc, argv);
+}
+
+
+bool
+select_history()
+{
+ if (opt_incremental_history_name || opt_incremental_history_uuid) {
+ if (!select_incremental_lsn_from_history(
+ &incremental_lsn)) {
+ return(false);
+ }
+ }
+ return(true);
+}
+
+bool
+flush_changed_page_bitmaps()
+{
+ if (xtrabackup_incremental && have_changed_page_bitmaps &&
+ !xtrabackup_incremental_force_scan) {
+ xb_mysql_query(mysql_connection,
+ "FLUSH NO_WRITE_TO_BINLOG CHANGED_PAGE_BITMAPS", false);
+ }
+ return(true);
+}
+
+
+/*********************************************************************//**
+Deallocate memory, disconnect from MySQL server, etc.
+@return true on success. */
+void
+backup_cleanup()
+{
+ free(mysql_slave_position);
+ free(mysql_binlog_position);
+ free(buffer_pool_filename);
+
+ if (mysql_connection) {
+ mysql_close(mysql_connection);
+ }
+}
+
+
+static pthread_mutex_t mdl_lock_con_mutex;
+static MYSQL *mdl_con = NULL;
+
+void
+mdl_lock_init()
+{
+ pthread_mutex_init(&mdl_lock_con_mutex, NULL);
+ mdl_con = xb_mysql_connect();
+ if (mdl_con)
+ {
+ xb_mysql_query(mdl_con, "BEGIN", false, true);
+ }
+}
+
+#ifndef DBUF_OFF
+/* Test that table is really locked, if lock_ddl_per_table is set.
+ The test is executed in DBUG_EXECUTE_IF block inside mdl_lock_table().
+*/
+static void check_mdl_lock_works(const char *table_name)
+{
+ MYSQL *test_con= xb_mysql_connect();
+ char *query;
+ xb_a(asprintf(&query,
+ "SET STATEMENT max_statement_time=1 FOR ALTER TABLE %s"
+ " ADD COLUMN mdl_lock_column int", table_name));
+ int err = mysql_query(test_con, query);
+ DBUG_ASSERT(err);
+ int err_no = mysql_errno(test_con);
+ DBUG_ASSERT(err_no == ER_STATEMENT_TIMEOUT);
+ mysql_close(test_con);
+ free(query);
+}
+#endif
+void
+mdl_lock_table(ulint space_id)
+{
+ std::ostringstream oss;
+ oss << "SELECT NAME "
+ "FROM INFORMATION_SCHEMA.INNODB_SYS_TABLES "
+ "WHERE SPACE = " << space_id << " AND NAME LIKE '%%/%%'";
+
+ pthread_mutex_lock(&mdl_lock_con_mutex);
+
+ MYSQL_RES *mysql_result = xb_mysql_query(mdl_con, oss.str().c_str(), true, true);
+
+ while (MYSQL_ROW row = mysql_fetch_row(mysql_result)) {
+ std::string full_table_name = ut_get_name(0,row[0]);
+ std::ostringstream lock_query;
+ lock_query << "SELECT * FROM " << full_table_name << " LIMIT 0";
+
+ msg_ts("Locking MDL for %s\n", full_table_name.c_str());
+ xb_mysql_query(mdl_con, lock_query.str().c_str(), false, false);
+
+ DBUG_EXECUTE_IF("check_mdl_lock_works",
+ check_mdl_lock_works(full_table_name.c_str()););
+ }
+
+ pthread_mutex_unlock(&mdl_lock_con_mutex);
+ mysql_free_result(mysql_result);
+}
+
+
+void
+mdl_unlock_all()
+{
+ msg_ts("Unlocking MDL for all tables\n");
+ xb_mysql_query(mdl_con, "COMMIT", false, true);
+ mysql_close(mdl_con);
+ pthread_mutex_destroy(&mdl_lock_con_mutex);
+}
+
diff --git a/extra/mariabackup/backup_mysql.h b/extra/mariabackup/backup_mysql.h
new file mode 100644
index 00000000000..3ccd7bdb613
--- /dev/null
+++ b/extra/mariabackup/backup_mysql.h
@@ -0,0 +1,92 @@
+#ifndef XTRABACKUP_BACKUP_MYSQL_H
+#define XTRABACKUP_BACKUP_MYSQL_H
+
+#include
+
+/* mysql flavor and version */
+enum mysql_flavor_t { FLAVOR_UNKNOWN, FLAVOR_MYSQL,
+ FLAVOR_PERCONA_SERVER, FLAVOR_MARIADB };
+extern mysql_flavor_t server_flavor;
+extern unsigned long mysql_server_version;
+
+/* server capabilities */
+extern bool have_changed_page_bitmaps;
+extern bool have_backup_locks;
+extern bool have_lock_wait_timeout;
+extern bool have_galera_enabled;
+extern bool have_flush_engine_logs;
+extern bool have_multi_threaded_slave;
+extern bool have_gtid_slave;
+
+
+/* History on server */
+extern time_t history_start_time;
+extern time_t history_end_time;
+extern time_t history_lock_time;
+
+
+extern bool sql_thread_started;
+extern char *mysql_slave_position;
+extern char *mysql_binlog_position;
+extern char *buffer_pool_filename;
+
+/** connection to mysql server */
+extern MYSQL *mysql_connection;
+
+void
+capture_tool_command(int argc, char **argv);
+
+bool
+select_history();
+
+bool
+flush_changed_page_bitmaps();
+
+void
+backup_cleanup();
+
+bool
+get_mysql_vars(MYSQL *connection);
+
+bool
+detect_mysql_capabilities_for_backup();
+
+MYSQL *
+xb_mysql_connect();
+
+MYSQL_RES *
+xb_mysql_query(MYSQL *connection, const char *query, bool use_result,
+ bool die_on_error = true);
+
+void
+unlock_all(MYSQL *connection);
+
+bool
+write_current_binlog_file(MYSQL *connection);
+
+bool
+write_binlog_info(MYSQL *connection);
+
+bool
+write_xtrabackup_info(MYSQL *connection);
+
+bool
+write_backup_config_file();
+
+bool
+lock_binlog_maybe(MYSQL *connection);
+
+bool
+lock_tables(MYSQL *connection);
+
+bool
+wait_for_safe_slave(MYSQL *connection);
+
+bool
+write_galera_info(MYSQL *connection);
+
+bool
+write_slave_info(MYSQL *connection);
+
+
+#endif
diff --git a/extra/mariabackup/backup_wsrep.h b/extra/mariabackup/backup_wsrep.h
new file mode 100644
index 00000000000..6537b304e12
--- /dev/null
+++ b/extra/mariabackup/backup_wsrep.h
@@ -0,0 +1,32 @@
+/******************************************************
+Percona XtraBackup: hot backup tool for InnoDB
+(c) 2009-2014 Percona LLC and/or its affiliates
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+*******************************************************/
+
+#ifndef BACKUP_WSREP_H
+#define BACKUP_WSREP_H
+
+/***********************************************************************
+Store Galera checkpoint info in the 'xtrabackup_galera_info' file, if that
+information is present in the trx system header. Otherwise, do nothing. */
+void
+xb_write_galera_info(bool incremental_prepare);
+/*==================*/
+
+#endif
diff --git a/extra/mariabackup/changed_page_bitmap.cc b/extra/mariabackup/changed_page_bitmap.cc
new file mode 100644
index 00000000000..ce769375a16
--- /dev/null
+++ b/extra/mariabackup/changed_page_bitmap.cc
@@ -0,0 +1,1045 @@
+/******************************************************
+XtraBackup: hot backup tool for InnoDB
+(c) 2009-2012 Percona Inc.
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+/* Changed page bitmap implementation */
+
+#include "changed_page_bitmap.h"
+
+#include "common.h"
+#include "xtrabackup.h"
+
+/* TODO: copy-pasted shared definitions from the XtraDB bitmap write code.
+Remove these on the first opportunity, i.e. single-binary XtraBackup. */
+
+/* log0online.h */
+
+/** Single bitmap file information */
+struct log_online_bitmap_file_t {
+ char name[FN_REFLEN]; /*!< Name with full path */
+ pfs_os_file_t file; /*!< Handle to opened file */
+ ib_uint64_t size; /*!< Size of the file */
+ ib_uint64_t offset; /*!< Offset of the next read,
+ or count of already-read bytes
+ */
+};
+
+/** A set of bitmap files containing some LSN range */
+struct log_online_bitmap_file_range_t {
+ size_t count; /*!< Number of files */
+ /*!< Dynamically-allocated array of info about individual files */
+ struct files_t {
+ char name[FN_REFLEN];/*!< Name of a file */
+ lsn_t start_lsn; /*!< Starting LSN of data in this
+ file */
+ ulong seq_num; /*!< Sequence number of this file */
+ } *files;
+};
+
+/* log0online.c */
+
+/** File name stem for bitmap files. */
+static const char* bmp_file_name_stem = "ib_modified_log_";
+
+/** The bitmap file block size in bytes. All writes will be multiples of this.
+ */
+enum {
+ MODIFIED_PAGE_BLOCK_SIZE = 4096
+};
+
+/** Offsets in a file bitmap block */
+enum {
+ MODIFIED_PAGE_IS_LAST_BLOCK = 0,/* 1 if last block in the current
+ write, 0 otherwise. */
+ MODIFIED_PAGE_START_LSN = 4, /* The starting tracked LSN of this and
+ other blocks in the same write */
+ MODIFIED_PAGE_END_LSN = 12, /* The ending tracked LSN of this and
+ other blocks in the same write */
+ MODIFIED_PAGE_SPACE_ID = 20, /* The space ID of tracked pages in
+ this block */
+ MODIFIED_PAGE_1ST_PAGE_ID = 24, /* The page ID of the first tracked
+ page in this block */
+ MODIFIED_PAGE_BLOCK_UNUSED_1 = 28,/* Unused in order to align the start
+ of bitmap at 8 byte boundary */
+ MODIFIED_PAGE_BLOCK_BITMAP = 32,/* Start of the bitmap itself */
+ MODIFIED_PAGE_BLOCK_UNUSED_2 = MODIFIED_PAGE_BLOCK_SIZE - 8,
+ /* Unused in order to align the end of
+ bitmap at 8 byte boundary */
+ MODIFIED_PAGE_BLOCK_CHECKSUM = MODIFIED_PAGE_BLOCK_SIZE - 4
+ /* The checksum of the current block */
+};
+
+/** Length of the bitmap data in a block */
+enum { MODIFIED_PAGE_BLOCK_BITMAP_LEN
+ = MODIFIED_PAGE_BLOCK_UNUSED_2 - MODIFIED_PAGE_BLOCK_BITMAP };
+
+/** Length of the bitmap data in a block in page ids */
+enum { MODIFIED_PAGE_BLOCK_ID_COUNT = MODIFIED_PAGE_BLOCK_BITMAP_LEN * 8 };
+
+typedef ib_uint64_t bitmap_word_t;
+
+/****************************************************************//**
+Calculate a bitmap block checksum. Algorithm borrowed from
+log_block_calc_checksum.
+@return checksum */
+UNIV_INLINE
+ulint
+log_online_calc_checksum(
+/*=====================*/
+ const byte* block); /*! p2
+*/
+static
+int
+log_online_compare_bmp_keys(
+/*========================*/
+ const void* p1, /*! k2_start_page ? 1 : 0;
+ }
+ return k1_space < k2_space ? -1 : 1;
+}
+
+/****************************************************************//**
+Calculate a bitmap block checksum. Algorithm borrowed from
+log_block_calc_checksum.
+@return checksum */
+UNIV_INLINE
+ulint
+log_online_calc_checksum(
+/*=====================*/
+ const byte* block) /*! 24) {
+
+ sh = 0;
+ }
+ }
+
+ return sum;
+}
+
+/****************************************************************//**
+Read one bitmap data page and check it for corruption.
+
+@return TRUE if page read OK, FALSE if I/O error */
+static
+ibool
+log_online_read_bitmap_page(
+/*========================*/
+ log_online_bitmap_file_t *bitmap_file, /*!size >= MODIFIED_PAGE_BLOCK_SIZE);
+ ut_a(bitmap_file->offset
+ <= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE);
+ ut_a(bitmap_file->offset % MODIFIED_PAGE_BLOCK_SIZE == 0);
+ success = os_file_read(IORequestRead,
+ bitmap_file->file, page, bitmap_file->offset,
+ MODIFIED_PAGE_BLOCK_SIZE);
+
+ if (UNIV_UNLIKELY(!success)) {
+
+ /* The following call prints an error message */
+ os_file_get_last_error(TRUE);
+ msg("InnoDB: Warning: failed reading changed page bitmap "
+ "file \'%s\'\n", bitmap_file->name);
+ return FALSE;
+ }
+
+ bitmap_file->offset += MODIFIED_PAGE_BLOCK_SIZE;
+ ut_ad(bitmap_file->offset <= bitmap_file->size);
+
+ checksum = mach_read_from_4(page + MODIFIED_PAGE_BLOCK_CHECKSUM);
+ actual_checksum = log_online_calc_checksum(page);
+ *checksum_ok = (checksum == actual_checksum);
+
+ return TRUE;
+}
+
+/*********************************************************************//**
+Check the name of a given file if it's a changed page bitmap file and
+return file sequence and start LSN name components if it is. If is not,
+the values of output parameters are undefined.
+
+@return TRUE if a given file is a changed page bitmap file. */
+static
+ibool
+log_online_is_bitmap_file(
+/*======================*/
+ const os_file_stat_t* file_info, /*!name) < OS_FILE_MAX_PATH);
+
+ return ((file_info->type == OS_FILE_TYPE_FILE
+ || file_info->type == OS_FILE_TYPE_LINK)
+ && (sscanf(file_info->name, "%[a-z_]%lu_" LSN_PF ".xdb", stem,
+ bitmap_file_seq_num, bitmap_file_start_lsn) == 3)
+ && (!strcmp(stem, bmp_file_name_stem)));
+}
+
+/*********************************************************************//**
+List the bitmap files in srv_data_home and setup their range that contains the
+specified LSN interval. This range, if non-empty, will start with a file that
+has the greatest LSN equal to or less than the start LSN and will include all
+the files up to the one with the greatest LSN less than the end LSN. Caller
+must free bitmap_files->files when done if bitmap_files set to non-NULL and
+this function returned TRUE. Field bitmap_files->count might be set to a
+larger value than the actual count of the files, and space for the unused array
+slots will be allocated but cleared to zeroes.
+
+@return TRUE if succeeded
+*/
+static
+ibool
+log_online_setup_bitmap_file_range(
+/*===============================*/
+ log_online_bitmap_file_range_t *bitmap_files, /*!= range_start);
+
+ bitmap_files->count = 0;
+ bitmap_files->files = NULL;
+
+ /* 1st pass: size the info array */
+
+ bitmap_dir = os_file_opendir(srv_data_home, FALSE);
+ if (UNIV_UNLIKELY(!bitmap_dir)) {
+
+ msg("InnoDB: Error: failed to open bitmap directory \'%s\'\n",
+ srv_data_home);
+ return FALSE;
+ }
+
+ while (!os_file_readdir_next_file(srv_data_home, bitmap_dir,
+ &bitmap_dir_file_info)) {
+
+ ulong file_seq_num;
+ lsn_t file_start_lsn;
+
+ if (!log_online_is_bitmap_file(&bitmap_dir_file_info,
+ &file_seq_num,
+ &file_start_lsn)
+ || file_start_lsn >= range_end) {
+
+ continue;
+ }
+
+ if (file_seq_num > last_file_seq_num) {
+
+ last_file_seq_num = file_seq_num;
+ }
+
+ if (file_start_lsn >= range_start
+ || file_start_lsn == first_file_start_lsn
+ || first_file_start_lsn > range_start) {
+
+ /* A file that falls into the range */
+
+ if (file_start_lsn < first_file_start_lsn) {
+
+ first_file_start_lsn = file_start_lsn;
+ }
+ if (file_seq_num < first_file_seq_num) {
+
+ first_file_seq_num = file_seq_num;
+ }
+ } else if (file_start_lsn > first_file_start_lsn) {
+
+ /* A file that has LSN closer to the range start
+ but smaller than it, replacing another such file */
+ first_file_start_lsn = file_start_lsn;
+ first_file_seq_num = file_seq_num;
+ }
+ }
+
+ if (UNIV_UNLIKELY(os_file_closedir(bitmap_dir))) {
+
+ os_file_get_last_error(TRUE);
+ msg("InnoDB: Error: cannot close \'%s\'\n",srv_data_home);
+ return FALSE;
+ }
+
+ if (first_file_seq_num == ULONG_MAX && last_file_seq_num == 0) {
+
+ bitmap_files->count = 0;
+ return TRUE;
+ }
+
+ bitmap_files->count = last_file_seq_num - first_file_seq_num + 1;
+
+ /* 2nd pass: get the file names in the file_seq_num order */
+
+ bitmap_dir = os_file_opendir(srv_data_home, FALSE);
+ if (UNIV_UNLIKELY(!bitmap_dir)) {
+
+ msg("InnoDB: Error: failed to open bitmap directory \'%s\'\n",
+ srv_data_home);
+ return FALSE;
+ }
+
+ bitmap_files->files =
+ static_cast
+ (malloc(bitmap_files->count * sizeof(bitmap_files->files[0])));
+ memset(bitmap_files->files, 0,
+ bitmap_files->count * sizeof(bitmap_files->files[0]));
+
+ while (!os_file_readdir_next_file(srv_data_home, bitmap_dir,
+ &bitmap_dir_file_info)) {
+
+ ulong file_seq_num;
+ lsn_t file_start_lsn;
+ size_t array_pos;
+
+ if (!log_online_is_bitmap_file(&bitmap_dir_file_info,
+ &file_seq_num,
+ &file_start_lsn)
+ || file_start_lsn >= range_end
+ || file_start_lsn < first_file_start_lsn) {
+
+ continue;
+ }
+
+ array_pos = file_seq_num - first_file_seq_num;
+ if (UNIV_UNLIKELY(array_pos >= bitmap_files->count)) {
+
+ msg("InnoDB: Error: inconsistent bitmap file "
+ "directory\n");
+ free(bitmap_files->files);
+ return FALSE;
+ }
+
+ if (file_seq_num > bitmap_files->files[array_pos].seq_num) {
+
+ bitmap_files->files[array_pos].seq_num = file_seq_num;
+ strncpy(bitmap_files->files[array_pos].name,
+ bitmap_dir_file_info.name, FN_REFLEN);
+ bitmap_files->files[array_pos].name[FN_REFLEN - 1]
+ = '\0';
+ bitmap_files->files[array_pos].start_lsn
+ = file_start_lsn;
+ }
+ }
+
+ if (UNIV_UNLIKELY(os_file_closedir(bitmap_dir))) {
+
+ os_file_get_last_error(TRUE);
+ msg("InnoDB: Error: cannot close \'%s\'\n", srv_data_home);
+ free(bitmap_files->files);
+ return FALSE;
+ }
+
+#ifdef UNIV_DEBUG
+ ut_ad(bitmap_files->files[0].seq_num == first_file_seq_num);
+
+ for (size_t i = 1; i < bitmap_files->count; i++) {
+ if (!bitmap_files->files[i].seq_num) {
+
+ break;
+ }
+ ut_ad(bitmap_files->files[i].seq_num
+ > bitmap_files->files[i - 1].seq_num);
+ ut_ad(bitmap_files->files[i].start_lsn
+ >= bitmap_files->files[i - 1].start_lsn);
+ }
+#endif
+
+ return TRUE;
+}
+
+/****************************************************************//**
+Open a bitmap file for reading.
+
+@return whether opened successfully */
+static
+bool
+log_online_open_bitmap_file_read_only(
+/*==================================*/
+ const char* name, /*!name, FN_REFLEN, "%s%s", srv_data_home, name);
+ bitmap_file->file = os_file_create_simple_no_error_handling(
+ 0, bitmap_file->name,
+ OS_FILE_OPEN, OS_FILE_READ_ONLY, true, &success);
+ if (UNIV_UNLIKELY(!success)) {
+
+ /* Here and below assume that bitmap file names do not
+ contain apostrophes, thus no need for ut_print_filename(). */
+ msg("InnoDB: Warning: error opening the changed page "
+ "bitmap \'%s\'\n", bitmap_file->name);
+ return success;
+ }
+
+ bitmap_file->size = os_file_get_size(bitmap_file->file);
+ bitmap_file->offset = 0;
+
+#ifdef UNIV_LINUX
+ posix_fadvise(bitmap_file->file, 0, 0, POSIX_FADV_SEQUENTIAL);
+ posix_fadvise(bitmap_file->file, 0, 0, POSIX_FADV_NOREUSE);
+#endif
+
+ return success;
+}
+
+/****************************************************************//**
+Diagnose one or both of the following situations if we read close to
+the end of bitmap file:
+1) Warn if the remainder of the file is less than one page.
+2) Error if we cannot read any more full pages but the last read page
+did not have the last-in-run flag set.
+
+@return FALSE for the error */
+static
+ibool
+log_online_diagnose_bitmap_eof(
+/*===========================*/
+ const log_online_bitmap_file_t* bitmap_file, /*!< in: bitmap file */
+ ibool last_page_in_run)/*!< in: "last page in
+ run" flag value in the
+ last read page */
+{
+ /* Check if we are too close to EOF to read a full page */
+ if ((bitmap_file->size < MODIFIED_PAGE_BLOCK_SIZE)
+ || (bitmap_file->offset
+ > bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE)) {
+
+ if (UNIV_UNLIKELY(bitmap_file->offset != bitmap_file->size)) {
+
+ /* If we are not at EOF and we have less than one page
+ to read, it's junk. This error is not fatal in
+ itself. */
+
+ msg("InnoDB: Warning: junk at the end of changed "
+ "page bitmap file \'%s\'.\n", bitmap_file->name);
+ }
+
+ if (UNIV_UNLIKELY(!last_page_in_run)) {
+
+ /* We are at EOF but the last read page did not finish
+ a run */
+ /* It's a "Warning" here because it's not a fatal error
+ for the whole server */
+ msg("InnoDB: Warning: changed page bitmap "
+ "file \'%s\' does not contain a complete run "
+ "at the end.\n", bitmap_file->name);
+ return FALSE;
+ }
+ }
+ return TRUE;
+}
+
+/* End of copy-pasted definitions */
+
+/** Iterator structure over changed page bitmap */
+struct xb_page_bitmap_range_struct {
+ const xb_page_bitmap *bitmap; /* Bitmap with data */
+ ulint space_id; /* Space id for this
+ iterator */
+ ulint bit_i; /* Bit index of the iterator
+ position in the current page */
+ const ib_rbt_node_t *bitmap_node; /* Current bitmap tree node */
+ const byte *bitmap_page; /* Current bitmap page */
+ ulint current_page_id;/* Current page id */
+};
+
+/****************************************************************//**
+Print a diagnostic message on missing bitmap data for an LSN range. */
+static
+void
+xb_msg_missing_lsn_data(
+/*====================*/
+ lsn_t missing_interval_start, /*!size >= MODIFIED_PAGE_BLOCK_SIZE);
+
+ *page_end_lsn = 0;
+
+ while ((*page_end_lsn <= lsn)
+ && (bitmap_file->offset
+ <= bitmap_file->size - MODIFIED_PAGE_BLOCK_SIZE)) {
+
+ next_to_last_page_ok = last_page_ok;
+ if (!log_online_read_bitmap_page(bitmap_file, page,
+ &last_page_ok)) {
+
+ return FALSE;
+ }
+
+ *page_end_lsn = mach_read_from_8(page + MODIFIED_PAGE_END_LSN);
+ }
+
+ /* We check two pages here because the last read page already contains
+ the required LSN data. If the next to the last one page is corrupted,
+ then we have no way of telling if that page contained the required LSN
+ range data too */
+ return last_page_ok && next_to_last_page_ok;
+}
+
+/****************************************************************//**
+Read the disk bitmap and build the changed page bitmap tree for the
+LSN interval incremental_lsn to checkpoint_lsn_start.
+
+@return the built bitmap tree or NULL if unable to read the full interval for
+any reason. */
+xb_page_bitmap*
+xb_page_bitmap_init(void)
+/*=====================*/
+{
+ log_online_bitmap_file_t bitmap_file;
+ lsn_t bmp_start_lsn = incremental_lsn;
+ lsn_t bmp_end_lsn = checkpoint_lsn_start;
+ byte page[MODIFIED_PAGE_BLOCK_SIZE];
+ lsn_t current_page_end_lsn;
+ xb_page_bitmap *result;
+ ibool last_page_in_run= FALSE;
+ log_online_bitmap_file_range_t bitmap_files;
+ size_t bmp_i;
+ ibool last_page_ok = TRUE;
+
+ if (UNIV_UNLIKELY(bmp_start_lsn > bmp_end_lsn)) {
+
+ msg("xtrabackup: incremental backup LSN " LSN_PF
+ " is larger than than the last checkpoint LSN " LSN_PF
+ "\n", bmp_start_lsn, bmp_end_lsn);
+ return NULL;
+ }
+
+ if (!log_online_setup_bitmap_file_range(&bitmap_files, bmp_start_lsn,
+ bmp_end_lsn)) {
+
+ return NULL;
+ }
+
+ /* Only accept no bitmap files returned if start LSN == end LSN */
+ if (bitmap_files.count == 0 && bmp_end_lsn != bmp_start_lsn) {
+
+ return NULL;
+ }
+
+ result = rbt_create(MODIFIED_PAGE_BLOCK_SIZE,
+ log_online_compare_bmp_keys);
+
+ if (bmp_start_lsn == bmp_end_lsn) {
+
+ /* Empty range - empty bitmap */
+ return result;
+ }
+
+ bmp_i = 0;
+
+ if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].start_lsn
+ > bmp_start_lsn)) {
+
+ /* The 1st file does not have the starting LSN data */
+ xb_msg_missing_lsn_data(bmp_start_lsn,
+ bitmap_files.files[bmp_i].start_lsn);
+ rbt_free(result);
+ free(bitmap_files.files);
+ return NULL;
+ }
+
+ /* Skip any zero-sized files at the start */
+ while ((bmp_i < bitmap_files.count - 1)
+ && (bitmap_files.files[bmp_i].start_lsn
+ == bitmap_files.files[bmp_i + 1].start_lsn)) {
+
+ bmp_i++;
+ }
+
+ /* Is the 1st bitmap file missing? */
+ if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].name[0] == '\0')) {
+
+ /* TODO: this is not the exact missing range */
+ xb_msg_missing_lsn_data(bmp_start_lsn, bmp_end_lsn);
+ rbt_free(result);
+ free(bitmap_files.files);
+ return NULL;
+ }
+
+ /* Open the 1st bitmap file */
+ if (UNIV_UNLIKELY(!log_online_open_bitmap_file_read_only(
+ bitmap_files.files[bmp_i].name,
+ &bitmap_file))) {
+
+ rbt_free(result);
+ free(bitmap_files.files);
+ return NULL;
+ }
+
+ /* If the 1st file is truncated, no data. Not merged with the case
+ below because zero-length file indicates not a corruption but missing
+ subsequent files instead. */
+ if (UNIV_UNLIKELY(bitmap_file.size < MODIFIED_PAGE_BLOCK_SIZE)) {
+
+ xb_msg_missing_lsn_data(bmp_start_lsn, bmp_end_lsn);
+ rbt_free(result);
+ free(bitmap_files.files);
+ os_file_close(bitmap_file.file);
+ return NULL;
+ }
+
+ /* Find the start of the required LSN range in the file */
+ if (UNIV_UNLIKELY(!xb_find_lsn_in_bitmap_file(&bitmap_file, page,
+ ¤t_page_end_lsn,
+ bmp_start_lsn))) {
+
+ msg("xtrabackup: Warning: changed page bitmap file "
+ "\'%s\' corrupted\n", bitmap_file.name);
+ rbt_free(result);
+ free(bitmap_files.files);
+ os_file_close(bitmap_file.file);
+ return NULL;
+ }
+
+ last_page_in_run
+ = mach_read_from_4(page + MODIFIED_PAGE_IS_LAST_BLOCK);
+
+ if (UNIV_UNLIKELY(!log_online_diagnose_bitmap_eof(&bitmap_file,
+ last_page_in_run))) {
+
+ rbt_free(result);
+ free(bitmap_files.files);
+ os_file_close(bitmap_file.file);
+ return NULL;
+ }
+
+ if (UNIV_UNLIKELY(current_page_end_lsn < bmp_start_lsn)) {
+
+ xb_msg_missing_lsn_data(current_page_end_lsn, bmp_start_lsn);
+ rbt_free(result);
+ free(bitmap_files.files);
+ os_file_close(bitmap_file.file);
+ return NULL;
+ }
+
+ /* 1st bitmap page found, add it to the tree. */
+ rbt_insert(result, page, page);
+
+ /* Read next pages/files until all required data is read */
+ while (last_page_ok
+ && (current_page_end_lsn < bmp_end_lsn
+ || (current_page_end_lsn == bmp_end_lsn
+ && !last_page_in_run))) {
+
+ ib_rbt_bound_t tree_search_pos;
+
+ /* If EOF, advance the file skipping over any empty files */
+ while (bitmap_file.size < MODIFIED_PAGE_BLOCK_SIZE
+ || (bitmap_file.offset
+ > bitmap_file.size - MODIFIED_PAGE_BLOCK_SIZE)) {
+
+ os_file_close(bitmap_file.file);
+
+ if (UNIV_UNLIKELY(
+ !log_online_diagnose_bitmap_eof(
+ &bitmap_file, last_page_in_run))) {
+
+ rbt_free(result);
+ free(bitmap_files.files);
+ return NULL;
+ }
+
+ bmp_i++;
+
+ if (UNIV_UNLIKELY(bmp_i == bitmap_files.count
+ || (bitmap_files.files[bmp_i].seq_num
+ == 0))) {
+
+ xb_msg_missing_lsn_data(current_page_end_lsn,
+ bmp_end_lsn);
+ rbt_free(result);
+ free(bitmap_files.files);
+ return NULL;
+ }
+
+ /* Is the next file missing? */
+ if (UNIV_UNLIKELY(bitmap_files.files[bmp_i].name[0]
+ == '\0')) {
+
+ /* TODO: this is not the exact missing range */
+ xb_msg_missing_lsn_data(bitmap_files.files
+ [bmp_i - 1].start_lsn,
+ bmp_end_lsn);
+ rbt_free(result);
+ free(bitmap_files.files);
+ return NULL;
+ }
+
+ if (UNIV_UNLIKELY(
+ !log_online_open_bitmap_file_read_only(
+ bitmap_files.files[bmp_i].name,
+ &bitmap_file))) {
+
+ rbt_free(result);
+ free(bitmap_files.files);
+ return NULL;
+ }
+ }
+
+ if (UNIV_UNLIKELY(
+ !log_online_read_bitmap_page(&bitmap_file, page,
+ &last_page_ok))) {
+
+ rbt_free(result);
+ free(bitmap_files.files);
+ os_file_close(bitmap_file.file);
+ return NULL;
+ }
+
+ if (UNIV_UNLIKELY(!last_page_ok)) {
+
+ msg("xtrabackup: warning: changed page bitmap file "
+ "\'%s\' corrupted.\n", bitmap_file.name);
+ rbt_free(result);
+ free(bitmap_files.files);
+ os_file_close(bitmap_file.file);
+ return NULL;
+ }
+
+ /* Merge the current page with an existing page or insert a new
+ page into the tree */
+
+ if (!rbt_search(result, &tree_search_pos, page)) {
+
+ /* Merge the bitmap pages */
+ byte *existing_page
+ = rbt_value(byte, tree_search_pos.last);
+ bitmap_word_t *bmp_word_1 = (bitmap_word_t *)
+ (existing_page + MODIFIED_PAGE_BLOCK_BITMAP);
+ bitmap_word_t *bmp_end = (bitmap_word_t *)
+ (existing_page + MODIFIED_PAGE_BLOCK_UNUSED_2);
+ bitmap_word_t *bmp_word_2 = (bitmap_word_t *)
+ (page + MODIFIED_PAGE_BLOCK_BITMAP);
+ while (bmp_word_1 < bmp_end) {
+
+ *bmp_word_1++ |= *bmp_word_2++;
+ }
+ xb_a (bmp_word_1 == bmp_end);
+ } else {
+
+ /* Add a new page */
+ rbt_add_node(result, &tree_search_pos, page);
+ }
+
+ current_page_end_lsn
+ = mach_read_from_8(page + MODIFIED_PAGE_END_LSN);
+ last_page_in_run
+ = mach_read_from_4(page + MODIFIED_PAGE_IS_LAST_BLOCK);
+ }
+
+ xb_a (current_page_end_lsn >= bmp_end_lsn);
+
+ free(bitmap_files.files);
+ os_file_close(bitmap_file.file);
+
+ return result;
+}
+
+/****************************************************************//**
+Free the bitmap tree. */
+void
+xb_page_bitmap_deinit(
+/*==================*/
+ xb_page_bitmap* bitmap) /*!bitmap_page has been
+already found/bumped by rbt_search()/rbt_next().
+
+@return FALSE if no more bitmap data for the range space ID */
+static
+ibool
+xb_page_bitmap_setup_next_page(
+/*===========================*/
+ xb_page_bitmap_range* bitmap_range) /*!bitmap_node == NULL) {
+
+ bitmap_range->current_page_id = ULINT_UNDEFINED;
+ return FALSE;
+ }
+
+ bitmap_range->bitmap_page = rbt_value(byte, bitmap_range->bitmap_node);
+
+ new_space_id = mach_read_from_4(bitmap_range->bitmap_page
+ + MODIFIED_PAGE_SPACE_ID);
+ if (new_space_id != bitmap_range->space_id) {
+
+ /* No more data for the current page id. */
+ xb_a(new_space_id > bitmap_range->space_id);
+ bitmap_range->current_page_id = ULINT_UNDEFINED;
+ return FALSE;
+ }
+
+ new_1st_page_id = mach_read_from_4(bitmap_range->bitmap_page +
+ MODIFIED_PAGE_1ST_PAGE_ID);
+ xb_a (new_1st_page_id >= bitmap_range->current_page_id
+ || bitmap_range->current_page_id == ULINT_UNDEFINED);
+
+ bitmap_range->current_page_id = new_1st_page_id;
+ bitmap_range->bit_i = 0;
+
+ return TRUE;
+}
+
+/** Find the node with the smallest key that greater than equal to search key.
+@param[in] tree red-black tree
+@param[in] key search key
+@return node with the smallest greater-than-or-equal key
+@retval NULL if none was found */
+static
+const ib_rbt_node_t*
+rbt_lower_bound(const ib_rbt_t* tree, const void* key)
+{
+ ut_ad(!tree->cmp_arg);
+ const ib_rbt_node_t* ge = NULL;
+
+ for (const ib_rbt_node_t *node = tree->root->left;
+ node != tree->nil; ) {
+ int result = tree->compare(node->value, key);
+
+ if (result < 0) {
+ node = node->right;
+ } else {
+ ge = node;
+ if (result == 0) {
+ break;
+ }
+
+ node = node->left;
+ }
+ }
+
+ return(ge);
+}
+
+/****************************************************************//**
+Set up a new bitmap range iterator over a given space id changed
+pages in a given bitmap.
+
+@return bitmap range iterator */
+xb_page_bitmap_range*
+xb_page_bitmap_range_init(
+/*======================*/
+ xb_page_bitmap* bitmap, /*!< in: bitmap to iterate over */
+ ulint space_id) /*!< in: space id */
+{
+ byte search_page[MODIFIED_PAGE_BLOCK_SIZE];
+ xb_page_bitmap_range *result
+ = static_cast(malloc(sizeof(*result)));
+
+ memset(result, 0, sizeof(*result));
+ result->bitmap = bitmap;
+ result->space_id = space_id;
+ result->current_page_id = ULINT_UNDEFINED;
+
+ /* Search for the 1st page for the given space id */
+ /* This also sets MODIFIED_PAGE_1ST_PAGE_ID to 0, which is what we
+ want. */
+ memset(search_page, 0, MODIFIED_PAGE_BLOCK_SIZE);
+ mach_write_to_4(search_page + MODIFIED_PAGE_SPACE_ID, space_id);
+
+ result->bitmap_node = rbt_lower_bound(result->bitmap, search_page);
+
+ xb_page_bitmap_setup_next_page(result);
+
+ return result;
+}
+
+/****************************************************************//**
+Get the value of the bitmap->range->bit_i bitmap bit
+
+@return the current bit value */
+static inline
+ibool
+is_bit_set(
+/*=======*/
+ const xb_page_bitmap_range* bitmap_range) /*!< in: bitmap
+ range */
+{
+ return ((*(((bitmap_word_t *)(bitmap_range->bitmap_page
+ + MODIFIED_PAGE_BLOCK_BITMAP))
+ + (bitmap_range->bit_i >> 6)))
+ & (1ULL << (bitmap_range->bit_i & 0x3F))) ? TRUE : FALSE;
+}
+
+/****************************************************************//**
+Get the next page id that has its bit set or cleared, i.e. equal to
+bit_value.
+
+@return page id */
+ulint
+xb_page_bitmap_range_get_next_bit(
+/*==============================*/
+ xb_page_bitmap_range* bitmap_range, /*!< in/out: bitmap range */
+ ibool bit_value) /*!< in: bit value */
+{
+ if (UNIV_UNLIKELY(bitmap_range->current_page_id
+ == ULINT_UNDEFINED)) {
+
+ return ULINT_UNDEFINED;
+ }
+
+ do {
+ while (bitmap_range->bit_i < MODIFIED_PAGE_BLOCK_ID_COUNT) {
+
+ while (is_bit_set(bitmap_range) != bit_value
+ && (bitmap_range->bit_i
+ < MODIFIED_PAGE_BLOCK_ID_COUNT)) {
+
+ bitmap_range->current_page_id++;
+ bitmap_range->bit_i++;
+ }
+
+ if (bitmap_range->bit_i
+ < MODIFIED_PAGE_BLOCK_ID_COUNT) {
+
+ ulint result = bitmap_range->current_page_id;
+ bitmap_range->current_page_id++;
+ bitmap_range->bit_i++;
+ return result;
+ }
+ }
+
+ bitmap_range->bitmap_node
+ = rbt_next(bitmap_range->bitmap,
+ bitmap_range->bitmap_node);
+
+ } while (xb_page_bitmap_setup_next_page(bitmap_range));
+
+ return ULINT_UNDEFINED;
+}
+
+/****************************************************************//**
+Free the bitmap range iterator. */
+void
+xb_page_bitmap_range_deinit(
+/*========================*/
+ xb_page_bitmap_range* bitmap_range) /*! in/out: bitmap range */
+{
+ free(bitmap_range);
+}
diff --git a/extra/mariabackup/changed_page_bitmap.h b/extra/mariabackup/changed_page_bitmap.h
new file mode 100644
index 00000000000..6f549f47400
--- /dev/null
+++ b/extra/mariabackup/changed_page_bitmap.h
@@ -0,0 +1,85 @@
+/******************************************************
+XtraBackup: hot backup tool for InnoDB
+(c) 2009-2012 Percona Inc.
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+/* Changed page bitmap interface */
+
+#ifndef XB_CHANGED_PAGE_BITMAP_H
+#define XB_CHANGED_PAGE_BITMAP_H
+
+#include
+#include
+
+/* The changed page bitmap structure */
+typedef ib_rbt_t xb_page_bitmap;
+
+struct xb_page_bitmap_range_struct;
+
+/* The bitmap range iterator over one space id */
+typedef struct xb_page_bitmap_range_struct xb_page_bitmap_range;
+
+/****************************************************************//**
+Read the disk bitmap and build the changed page bitmap tree for the
+LSN interval incremental_lsn to checkpoint_lsn_start.
+
+@return the built bitmap tree */
+xb_page_bitmap*
+xb_page_bitmap_init(void);
+/*=====================*/
+
+/****************************************************************//**
+Free the bitmap tree. */
+void
+xb_page_bitmap_deinit(
+/*==================*/
+ xb_page_bitmap* bitmap); /*!
+#include
+#include
+#include
+
+
+# define fil_is_user_tablespace_id(i) ((i) > srv_undo_tablespaces_open)
+
+#ifdef _MSC_VER
+#define stat _stati64
+#define PATH_MAX MAX_PATH
+#endif
+
+#ifndef HAVE_VASPRINTF
+static inline int vasprintf(char **strp, const char *fmt, va_list args)
+{
+ int len;
+#ifdef _MSC_VER
+ len = _vscprintf(fmt, args);
+#else
+ len = vsnprintf(NULL, 0, fmt, args);
+#endif
+ if (len < 0)
+ {
+ return -1;
+ }
+ *strp = (char *)malloc(len + 1);
+ if (!*strp)
+ {
+ return -1;
+ }
+ vsprintf(*strp, fmt, args);
+ return len;
+}
+
+static inline int asprintf(char **strp, const char *fmt,...)
+{
+ va_list args;
+ va_start(args, fmt);
+ int len = vasprintf(strp, fmt, args);
+ va_end(args);
+ return len;
+}
+#endif
+
+#define xb_a(expr) \
+ do { \
+ if (!(expr)) { \
+ msg("Assertion \"%s\" failed at %s:%lu\n", \
+ #expr, __FILE__, (ulong) __LINE__); \
+ abort(); \
+ } \
+ } while (0);
+
+#ifdef XB_DEBUG
+#define xb_ad(expr) xb_a(expr)
+#else
+#define xb_ad(expr)
+#endif
+
+#define XB_DELTA_INFO_SUFFIX ".meta"
+
+static inline int msg(const char *fmt, ...) ATTRIBUTE_FORMAT(printf, 1, 2);
+static inline int msg(const char *fmt, ...)
+{
+ int result;
+ va_list args;
+
+ va_start(args, fmt);
+ result = vfprintf(stderr, fmt, args);
+ va_end(args);
+
+ return result;
+}
+
+static inline int msg_ts(const char *fmt, ...) ATTRIBUTE_FORMAT(printf, 1, 2);
+static inline int msg_ts(const char *fmt, ...)
+{
+ int result;
+ time_t t = time(NULL);
+ char date[100];
+ char *line;
+ va_list args;
+
+ strftime(date, sizeof(date), "%y%m%d %H:%M:%S", localtime(&t));
+
+ va_start(args, fmt);
+ result = vasprintf(&line, fmt, args);
+ va_end(args);
+
+ if (result != -1) {
+ result = fprintf(stderr, "%s %s", date, line);
+ free(line);
+ }
+
+ return result;
+}
+
+/* Use POSIX_FADV_NORMAL when available */
+
+#ifdef POSIX_FADV_NORMAL
+# define USE_POSIX_FADVISE
+#else
+# define POSIX_FADV_NORMAL
+# define POSIX_FADV_SEQUENTIAL
+# define POSIX_FADV_DONTNEED
+# define posix_fadvise(a,b,c,d) do {} while(0)
+#endif
+
+/***********************************************************************
+Computes bit shift for a given value. If the argument is not a power
+of 2, returns 0.*/
+static inline size_t
+get_bit_shift(size_t value)
+{
+ size_t shift;
+
+ if (value == 0)
+ return 0;
+
+ for (shift = 0; !(value & 1); shift++) {
+ value >>= 1;
+ }
+ return (value >> 1) ? 0 : shift;
+}
+
+/****************************************************************************
+Read 'len' bytes from 'fd'. It is identical to my_read(..., MYF(MY_FULL_IO)),
+i.e. tries to combine partial reads into a single block of size 'len', except
+that it bails out on EOF or error, and returns the number of successfully read
+bytes instead. */
+static inline size_t
+xb_read_full(File fd, uchar *buf, size_t len)
+{
+ size_t tlen = 0;
+ size_t tbytes;
+
+ while (tlen < len) {
+ tbytes = my_read(fd, buf, len - tlen, MYF(MY_WME));
+ if (tbytes == 0 || tbytes == MY_FILE_ERROR) {
+ break;
+ }
+
+ buf += tbytes;
+ tlen += tbytes;
+ }
+
+ return tlen;
+}
+
+#endif
diff --git a/BUILD/compile-pentium-myodbc b/extra/mariabackup/crc/CMakeLists.txt
old mode 100755
new mode 100644
similarity index 52%
rename from BUILD/compile-pentium-myodbc
rename to extra/mariabackup/crc/CMakeLists.txt
index d9b1dd47129..91758cdf520
--- a/BUILD/compile-pentium-myodbc
+++ b/extra/mariabackup/crc/CMakeLists.txt
@@ -1,28 +1,33 @@
-#! /bin/sh
-
-# Copyright (C) 2000 MySQL AB
-# Use is subject to license terms
+# Copyright (c) 2017 Percona LLC and/or its affiliates.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
-#
+
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
-#
+
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
-path=`dirname $0`
-. "$path/SETUP.sh"
+PROJECT(crc C)
-extra_flags="$pentium_cflags $fast_cflags"
-extra_configs="$pentium_configs --without-server"
-
-make=no
-strip=yes
-
-. "$path/FINISH.sh"
+IF(NOT CMAKE_CROSSCOMPILING AND NOT MSVC)
+ STRING(TOLOWER ${CMAKE_SYSTEM_PROCESSOR} processor)
+ IF(processor MATCHES "86" OR processor MATCHES "amd64" OR processor MATCHES "x64")
+ # Check for PCLMUL instruction
+ CHECK_C_SOURCE_RUNS("
+ int main()
+ {
+ asm volatile (\"pclmulqdq \\$0x00, %%xmm1, %%xmm0\":::\"cc\");
+ return 0;
+ }" HAVE_CLMUL_INSTRUCTION)
+ ENDIF()
+ENDIF()
+IF(HAVE_CLMUL_INSTRUCTION)
+ ADD_DEFINITIONS(-DHAVE_CLMUL_INSTRUCTION)
+ENDIF()
+ADD_LIBRARY(crc STATIC crc_glue.c crc-intel-pclmul.c)
diff --git a/extra/mariabackup/crc/config.h.cmake b/extra/mariabackup/crc/config.h.cmake
new file mode 100644
index 00000000000..fe81c1859ae
--- /dev/null
+++ b/extra/mariabackup/crc/config.h.cmake
@@ -0,0 +1,21 @@
+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+Zlib compatible CRC-32 implementation.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#cmakedefine HAVE_CLMUL_INSTRUCTION 1
diff --git a/extra/mariabackup/crc/crc-intel-pclmul.c b/extra/mariabackup/crc/crc-intel-pclmul.c
new file mode 100644
index 00000000000..d470c2bee43
--- /dev/null
+++ b/extra/mariabackup/crc/crc-intel-pclmul.c
@@ -0,0 +1,511 @@
+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+CRC32 using Intel's PCLMUL instruction.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+/* crc-intel-pclmul.c - Intel PCLMUL accelerated CRC implementation
+ * Copyright (C) 2016 Jussi Kivilinna
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA
+ *
+ */
+
+#include
+#include
+#include
+#include
+
+# define U64_C(c) (c ## UL)
+
+typedef uint32_t u32;
+typedef uint16_t u16;
+typedef uint64_t u64;
+#ifndef byte
+typedef uint8_t byte;
+#endif
+
+# define _gcry_bswap32 __builtin_bswap32
+
+#if __GNUC__ >= 4 && defined(__x86_64__) && defined(HAVE_CLMUL_INSTRUCTION)
+
+#if _GCRY_GCC_VERSION >= 40400 /* 4.4 */
+/* Prevent compiler from issuing SSE instructions between asm blocks. */
+# pragma GCC target("no-sse")
+#endif
+
+
+#define ALIGNED_16 __attribute__ ((aligned (16)))
+
+
+struct u16_unaligned_s
+{
+ u16 a;
+} __attribute__((packed, aligned (1), may_alias));
+
+
+/* Constants structure for generic reflected/non-reflected CRC32 CLMUL
+ * functions. */
+struct crc32_consts_s
+{
+ /* k: { x^(32*17), x^(32*15), x^(32*5), x^(32*3), x^(32*2), 0 } mod P(x) */
+ u64 k[6];
+ /* my_p: { floor(x^64 / P(x)), P(x) } */
+ u64 my_p[2];
+};
+
+
+/* CLMUL constants for CRC32 and CRC32RFC1510. */
+static const struct crc32_consts_s crc32_consts ALIGNED_16 =
+{
+ { /* k[6] = reverse_33bits( x^(32*y) mod P(x) ) */
+ U64_C(0x154442bd4), U64_C(0x1c6e41596), /* y = { 17, 15 } */
+ U64_C(0x1751997d0), U64_C(0x0ccaa009e), /* y = { 5, 3 } */
+ U64_C(0x163cd6124), 0 /* y = 2 */
+ },
+ { /* my_p[2] = reverse_33bits ( { floor(x^64 / P(x)), P(x) } ) */
+ U64_C(0x1f7011641), U64_C(0x1db710641)
+ }
+};
+
+/* Common constants for CRC32 algorithms. */
+static const byte crc32_refl_shuf_shift[3 * 16] ALIGNED_16 =
+ {
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+ 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ };
+static const byte crc32_partial_fold_input_mask[16 + 16] ALIGNED_16 =
+ {
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ };
+static const u64 crc32_merge9to15_shuf[15 - 9 + 1][2] ALIGNED_16 =
+ {
+ { U64_C(0x0706050403020100), U64_C(0xffffffffffffff0f) }, /* 9 */
+ { U64_C(0x0706050403020100), U64_C(0xffffffffffff0f0e) },
+ { U64_C(0x0706050403020100), U64_C(0xffffffffff0f0e0d) },
+ { U64_C(0x0706050403020100), U64_C(0xffffffff0f0e0d0c) },
+ { U64_C(0x0706050403020100), U64_C(0xffffff0f0e0d0c0b) },
+ { U64_C(0x0706050403020100), U64_C(0xffff0f0e0d0c0b0a) },
+ { U64_C(0x0706050403020100), U64_C(0xff0f0e0d0c0b0a09) }, /* 15 */
+ };
+static const u64 crc32_merge5to7_shuf[7 - 5 + 1][2] ALIGNED_16 =
+ {
+ { U64_C(0xffffff0703020100), U64_C(0xffffffffffffffff) }, /* 5 */
+ { U64_C(0xffff070603020100), U64_C(0xffffffffffffffff) },
+ { U64_C(0xff07060503020100), U64_C(0xffffffffffffffff) }, /* 7 */
+ };
+
+/* PCLMUL functions for reflected CRC32. */
+static inline void
+crc32_reflected_bulk (u32 *pcrc, const byte *inbuf, size_t inlen,
+ const struct crc32_consts_s *consts)
+{
+ if (inlen >= 8 * 16)
+ {
+ asm volatile ("movd %[crc], %%xmm4\n\t"
+ "movdqu %[inbuf_0], %%xmm0\n\t"
+ "movdqu %[inbuf_1], %%xmm1\n\t"
+ "movdqu %[inbuf_2], %%xmm2\n\t"
+ "movdqu %[inbuf_3], %%xmm3\n\t"
+ "pxor %%xmm4, %%xmm0\n\t"
+ :
+ : [inbuf_0] "m" (inbuf[0 * 16]),
+ [inbuf_1] "m" (inbuf[1 * 16]),
+ [inbuf_2] "m" (inbuf[2 * 16]),
+ [inbuf_3] "m" (inbuf[3 * 16]),
+ [crc] "m" (*pcrc)
+ );
+
+ inbuf += 4 * 16;
+ inlen -= 4 * 16;
+
+ asm volatile ("movdqa %[k1k2], %%xmm4\n\t"
+ :
+ : [k1k2] "m" (consts->k[1 - 1])
+ );
+
+ /* Fold by 4. */
+ while (inlen >= 4 * 16)
+ {
+ asm volatile ("movdqu %[inbuf_0], %%xmm5\n\t"
+ "movdqa %%xmm0, %%xmm6\n\t"
+ "pclmulqdq $0x00, %%xmm4, %%xmm0\n\t"
+ "pclmulqdq $0x11, %%xmm4, %%xmm6\n\t"
+ "pxor %%xmm5, %%xmm0\n\t"
+ "pxor %%xmm6, %%xmm0\n\t"
+
+ "movdqu %[inbuf_1], %%xmm5\n\t"
+ "movdqa %%xmm1, %%xmm6\n\t"
+ "pclmulqdq $0x00, %%xmm4, %%xmm1\n\t"
+ "pclmulqdq $0x11, %%xmm4, %%xmm6\n\t"
+ "pxor %%xmm5, %%xmm1\n\t"
+ "pxor %%xmm6, %%xmm1\n\t"
+
+ "movdqu %[inbuf_2], %%xmm5\n\t"
+ "movdqa %%xmm2, %%xmm6\n\t"
+ "pclmulqdq $0x00, %%xmm4, %%xmm2\n\t"
+ "pclmulqdq $0x11, %%xmm4, %%xmm6\n\t"
+ "pxor %%xmm5, %%xmm2\n\t"
+ "pxor %%xmm6, %%xmm2\n\t"
+
+ "movdqu %[inbuf_3], %%xmm5\n\t"
+ "movdqa %%xmm3, %%xmm6\n\t"
+ "pclmulqdq $0x00, %%xmm4, %%xmm3\n\t"
+ "pclmulqdq $0x11, %%xmm4, %%xmm6\n\t"
+ "pxor %%xmm5, %%xmm3\n\t"
+ "pxor %%xmm6, %%xmm3\n\t"
+ :
+ : [inbuf_0] "m" (inbuf[0 * 16]),
+ [inbuf_1] "m" (inbuf[1 * 16]),
+ [inbuf_2] "m" (inbuf[2 * 16]),
+ [inbuf_3] "m" (inbuf[3 * 16])
+ );
+
+ inbuf += 4 * 16;
+ inlen -= 4 * 16;
+ }
+
+ asm volatile ("movdqa %[k3k4], %%xmm6\n\t"
+ "movdqa %[my_p], %%xmm5\n\t"
+ :
+ : [k3k4] "m" (consts->k[3 - 1]),
+ [my_p] "m" (consts->my_p[0])
+ );
+
+ /* Fold 4 to 1. */
+
+ asm volatile ("movdqa %%xmm0, %%xmm4\n\t"
+ "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
+ "pclmulqdq $0x11, %%xmm6, %%xmm4\n\t"
+ "pxor %%xmm1, %%xmm0\n\t"
+ "pxor %%xmm4, %%xmm0\n\t"
+
+ "movdqa %%xmm0, %%xmm4\n\t"
+ "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
+ "pclmulqdq $0x11, %%xmm6, %%xmm4\n\t"
+ "pxor %%xmm2, %%xmm0\n\t"
+ "pxor %%xmm4, %%xmm0\n\t"
+
+ "movdqa %%xmm0, %%xmm4\n\t"
+ "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
+ "pclmulqdq $0x11, %%xmm6, %%xmm4\n\t"
+ "pxor %%xmm3, %%xmm0\n\t"
+ "pxor %%xmm4, %%xmm0\n\t"
+ :
+ :
+ );
+ }
+ else
+ {
+ asm volatile ("movd %[crc], %%xmm1\n\t"
+ "movdqu %[inbuf], %%xmm0\n\t"
+ "movdqa %[k3k4], %%xmm6\n\t"
+ "pxor %%xmm1, %%xmm0\n\t"
+ "movdqa %[my_p], %%xmm5\n\t"
+ :
+ : [inbuf] "m" (*inbuf),
+ [crc] "m" (*pcrc),
+ [k3k4] "m" (consts->k[3 - 1]),
+ [my_p] "m" (consts->my_p[0])
+ );
+
+ inbuf += 16;
+ inlen -= 16;
+ }
+
+ /* Fold by 1. */
+ if (inlen >= 16)
+ {
+ while (inlen >= 16)
+ {
+ /* Load next block to XMM2. Fold XMM0 to XMM0:XMM1. */
+ asm volatile ("movdqu %[inbuf], %%xmm2\n\t"
+ "movdqa %%xmm0, %%xmm1\n\t"
+ "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
+ "pclmulqdq $0x11, %%xmm6, %%xmm1\n\t"
+ "pxor %%xmm2, %%xmm0\n\t"
+ "pxor %%xmm1, %%xmm0\n\t"
+ :
+ : [inbuf] "m" (*inbuf)
+ );
+
+ inbuf += 16;
+ inlen -= 16;
+ }
+ }
+
+ /* Partial fold. */
+ if (inlen)
+ {
+ /* Load last input and add padding zeros. */
+ asm volatile ("movdqu %[shr_shuf], %%xmm3\n\t"
+ "movdqu %[shl_shuf], %%xmm4\n\t"
+ "movdqu %[mask], %%xmm2\n\t"
+
+ "movdqa %%xmm0, %%xmm1\n\t"
+ "pshufb %%xmm4, %%xmm0\n\t"
+ "movdqu %[inbuf], %%xmm4\n\t"
+ "pshufb %%xmm3, %%xmm1\n\t"
+ "pand %%xmm4, %%xmm2\n\t"
+ "por %%xmm1, %%xmm2\n\t"
+
+ "movdqa %%xmm0, %%xmm1\n\t"
+ "pclmulqdq $0x00, %%xmm6, %%xmm0\n\t"
+ "pclmulqdq $0x11, %%xmm6, %%xmm1\n\t"
+ "pxor %%xmm2, %%xmm0\n\t"
+ "pxor %%xmm1, %%xmm0\n\t"
+ :
+ : [inbuf] "m" (*(inbuf - 16 + inlen)),
+ [mask] "m" (crc32_partial_fold_input_mask[inlen]),
+ [shl_shuf] "m" (crc32_refl_shuf_shift[inlen]),
+ [shr_shuf] "m" (crc32_refl_shuf_shift[inlen + 16])
+ );
+
+ inbuf += inlen;
+ inlen -= inlen;
+ }
+
+ /* Final fold. */
+ asm volatile (/* reduce 128-bits to 96-bits */
+ "movdqa %%xmm0, %%xmm1\n\t"
+ "pclmulqdq $0x10, %%xmm6, %%xmm0\n\t"
+ "psrldq $8, %%xmm1\n\t"
+ "pxor %%xmm1, %%xmm0\n\t"
+
+ /* reduce 96-bits to 64-bits */
+ "pshufd $0xfc, %%xmm0, %%xmm1\n\t" /* [00][00][00][x] */
+ "pshufd $0xf9, %%xmm0, %%xmm0\n\t" /* [00][00][x>>64][x>>32] */
+ "pclmulqdq $0x00, %[k5], %%xmm1\n\t" /* [00][00][xx][xx] */
+ "pxor %%xmm1, %%xmm0\n\t" /* top 64-bit are zero */
+
+ /* barrett reduction */
+ "pshufd $0xf3, %%xmm0, %%xmm1\n\t" /* [00][00][x>>32][00] */
+ "pslldq $4, %%xmm0\n\t" /* [??][x>>32][??][??] */
+ "pclmulqdq $0x00, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */
+ "pclmulqdq $0x10, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */
+ "pxor %%xmm1, %%xmm0\n\t"
+
+ /* store CRC */
+ "pextrd $2, %%xmm0, %[out]\n\t"
+ : [out] "=m" (*pcrc)
+ : [k5] "m" (consts->k[5 - 1])
+ );
+}
+
+static inline void
+crc32_reflected_less_than_16 (u32 *pcrc, const byte *inbuf, size_t inlen,
+ const struct crc32_consts_s *consts)
+{
+ if (inlen < 4)
+ {
+ u32 crc = *pcrc;
+ u32 data;
+
+ asm volatile ("movdqa %[my_p], %%xmm5\n\t"
+ :
+ : [my_p] "m" (consts->my_p[0])
+ );
+
+ if (inlen == 1)
+ {
+ data = inbuf[0];
+ data ^= crc;
+ data <<= 24;
+ crc >>= 8;
+ }
+ else if (inlen == 2)
+ {
+ data = ((const struct u16_unaligned_s *)inbuf)->a;
+ data ^= crc;
+ data <<= 16;
+ crc >>= 16;
+ }
+ else
+ {
+ data = ((const struct u16_unaligned_s *)inbuf)->a;
+ data |= inbuf[2] << 16;
+ data ^= crc;
+ data <<= 8;
+ crc >>= 24;
+ }
+
+ /* Barrett reduction */
+ asm volatile ("movd %[in], %%xmm0\n\t"
+ "movd %[crc], %%xmm1\n\t"
+
+ "pclmulqdq $0x00, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */
+ "psllq $32, %%xmm1\n\t"
+ "pshufd $0xfc, %%xmm0, %%xmm0\n\t" /* [00][00][00][x] */
+ "pclmulqdq $0x10, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */
+ "pxor %%xmm1, %%xmm0\n\t"
+
+ "pextrd $1, %%xmm0, %[out]\n\t"
+ : [out] "=m" (*pcrc)
+ : [in] "rm" (data),
+ [crc] "rm" (crc)
+ );
+ }
+ else if (inlen == 4)
+ {
+ /* Barrett reduction */
+ asm volatile ("movd %[crc], %%xmm1\n\t"
+ "movd %[in], %%xmm0\n\t"
+ "movdqa %[my_p], %%xmm5\n\t"
+ "pxor %%xmm1, %%xmm0\n\t"
+
+ "pclmulqdq $0x00, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */
+ "pshufd $0xfc, %%xmm0, %%xmm0\n\t" /* [00][00][00][x] */
+ "pclmulqdq $0x10, %%xmm5, %%xmm0\n\t" /* [00][00][xx][xx] */
+
+ "pextrd $1, %%xmm0, %[out]\n\t"
+ : [out] "=m" (*pcrc)
+ : [in] "m" (*inbuf),
+ [crc] "m" (*pcrc),
+ [my_p] "m" (consts->my_p[0])
+ );
+ }
+ else
+ {
+ asm volatile ("movdqu %[shuf], %%xmm4\n\t"
+ "movd %[crc], %%xmm1\n\t"
+ "movdqa %[my_p], %%xmm5\n\t"
+ "movdqa %[k3k4], %%xmm6\n\t"
+ :
+ : [shuf] "m" (crc32_refl_shuf_shift[inlen]),
+ [crc] "m" (*pcrc),
+ [my_p] "m" (consts->my_p[0]),
+ [k3k4] "m" (consts->k[3 - 1])
+ );
+
+ if (inlen >= 8)
+ {
+ asm volatile ("movq %[inbuf], %%xmm0\n\t"
+ :
+ : [inbuf] "m" (*inbuf)
+ );
+ if (inlen > 8)
+ {
+ asm volatile (/*"pinsrq $1, %[inbuf_tail], %%xmm0\n\t"*/
+ "movq %[inbuf_tail], %%xmm2\n\t"
+ "punpcklqdq %%xmm2, %%xmm0\n\t"
+ "pshufb %[merge_shuf], %%xmm0\n\t"
+ :
+ : [inbuf_tail] "m" (inbuf[inlen - 8]),
+ [merge_shuf] "m"
+ (*crc32_merge9to15_shuf[inlen - 9])
+ );
+ }
+ }
+ else
+ {
+ asm volatile ("movd %[inbuf], %%xmm0\n\t"
+ "pinsrd $1, %[inbuf_tail], %%xmm0\n\t"
+ "pshufb %[merge_shuf], %%xmm0\n\t"
+ :
+ : [inbuf] "m" (*inbuf),
+ [inbuf_tail] "m" (inbuf[inlen - 4]),
+ [merge_shuf] "m"
+ (*crc32_merge5to7_shuf[inlen - 5])
+ );
+ }
+
+ /* Final fold. */
+ asm volatile ("pxor %%xmm1, %%xmm0\n\t"
+ "pshufb %%xmm4, %%xmm0\n\t"
+
+ /* reduce 128-bits to 96-bits */
+ "movdqa %%xmm0, %%xmm1\n\t"
+ "pclmulqdq $0x10, %%xmm6, %%xmm0\n\t"
+ "psrldq $8, %%xmm1\n\t"
+ "pxor %%xmm1, %%xmm0\n\t" /* top 32-bit are zero */
+
+ /* reduce 96-bits to 64-bits */
+ "pshufd $0xfc, %%xmm0, %%xmm1\n\t" /* [00][00][00][x] */
+ "pshufd $0xf9, %%xmm0, %%xmm0\n\t" /* [00][00][x>>64][x>>32] */
+ "pclmulqdq $0x00, %[k5], %%xmm1\n\t" /* [00][00][xx][xx] */
+ "pxor %%xmm1, %%xmm0\n\t" /* top 64-bit are zero */
+
+ /* barrett reduction */
+ "pshufd $0xf3, %%xmm0, %%xmm1\n\t" /* [00][00][x>>32][00] */
+ "pslldq $4, %%xmm0\n\t" /* [??][x>>32][??][??] */
+ "pclmulqdq $0x00, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */
+ "pclmulqdq $0x10, %%xmm5, %%xmm1\n\t" /* [00][xx][xx][00] */
+ "pxor %%xmm1, %%xmm0\n\t"
+
+ /* store CRC */
+ "pextrd $2, %%xmm0, %[out]\n\t"
+ : [out] "=m" (*pcrc)
+ : [k5] "m" (consts->k[5 - 1])
+ );
+ }
+}
+
+void
+crc32_intel_pclmul (u32 *pcrc, const byte *inbuf, size_t inlen)
+{
+ const struct crc32_consts_s *consts = &crc32_consts;
+#if defined(__x86_64__) && defined(__WIN64__)
+ char win64tmp[2 * 16];
+
+ /* XMM6-XMM7 need to be restored after use. */
+ asm volatile ("movdqu %%xmm6, 0*16(%0)\n\t"
+ "movdqu %%xmm7, 1*16(%0)\n\t"
+ :
+ : "r" (win64tmp)
+ : "memory");
+#endif
+
+ if (!inlen)
+ return;
+
+ if (inlen >= 16)
+ crc32_reflected_bulk(pcrc, inbuf, inlen, consts);
+ else
+ crc32_reflected_less_than_16(pcrc, inbuf, inlen, consts);
+
+#if defined(__x86_64__) && defined(__WIN64__)
+ /* Restore used registers. */
+ asm volatile("movdqu 0*16(%0), %%xmm6\n\t"
+ "movdqu 1*16(%0), %%xmm7\n\t"
+ :
+ : "r" (win64tmp)
+ : "memory");
+#endif
+}
+
+#endif
diff --git a/extra/mariabackup/crc/crc-intel-pclmul.h b/extra/mariabackup/crc/crc-intel-pclmul.h
new file mode 100644
index 00000000000..120058165a0
--- /dev/null
+++ b/extra/mariabackup/crc/crc-intel-pclmul.h
@@ -0,0 +1,25 @@
+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+CRC32 using Intel's PCLMUL instruction.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#include
+#include
+
+void
+crc32_intel_pclmul(uint32_t *pcrc, const uint8_t *inbuf, size_t inlen);
diff --git a/extra/mariabackup/crc/crc_glue.c b/extra/mariabackup/crc/crc_glue.c
new file mode 100644
index 00000000000..c301cb01e2e
--- /dev/null
+++ b/extra/mariabackup/crc/crc_glue.c
@@ -0,0 +1,72 @@
+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+Zlib compatible CRC-32 implementation.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+#include "my_config.h"
+#include "crc_glue.h"
+#include "crc-intel-pclmul.h"
+#include
+#include
+#include
+
+#if __GNUC__ >= 4 && defined(__x86_64__)
+static int pclmul_enabled = 0;
+#endif
+
+#if defined(__GNUC__) && defined(__x86_64__)
+static
+uint32_t
+cpuid(uint32_t* ecx, uint32_t* edx)
+{
+ uint32_t level;
+
+ asm("cpuid" : "=a" (level) : "a" (0) : "ebx", "ecx", "edx");
+
+ if (level < 1) {
+ return level;
+ }
+
+ asm("cpuid" : "=c" (*ecx), "=d" (*edx)
+ : "a" (1)
+ : "ebx");
+
+ return level;
+}
+#endif
+
+void crc_init() {
+#if defined(__GNUC__) && defined(__x86_64__)
+ uint32_t ecx, edx;
+
+ if (cpuid(&ecx, &edx) > 0) {
+ pclmul_enabled = ((ecx >> 19) & 1) && ((ecx >> 1) & 1);
+ }
+#endif
+}
+
+unsigned long crc32_iso3309(unsigned long crc, const unsigned char *buf, unsigned int len)
+{
+#if __GNUC__ >= 4 && defined(__x86_64__) && defined(HAVE_CLMUL_INSTRUCTION)
+ if (pclmul_enabled) {
+ uint32_t crc_accum = crc ^ 0xffffffffL;
+ crc32_intel_pclmul(&crc_accum, buf, len);
+ return crc_accum ^ 0xffffffffL;
+ }
+#endif
+ return crc32(crc, buf, len);
+}
diff --git a/extra/mariabackup/crc/crc_glue.h b/extra/mariabackup/crc/crc_glue.h
new file mode 100644
index 00000000000..e287fa4a7aa
--- /dev/null
+++ b/extra/mariabackup/crc/crc_glue.h
@@ -0,0 +1,31 @@
+/******************************************************
+Copyright (c) 2017 Percona LLC and/or its affiliates.
+
+Zlib compatible CRC-32 implementation.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void crc_init();
+unsigned long crc32_iso3309(unsigned long crc, const unsigned char *buf, unsigned int len);
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/extra/mariabackup/datasink.c b/extra/mariabackup/datasink.c
new file mode 100644
index 00000000000..460e0e8ca19
--- /dev/null
+++ b/extra/mariabackup/datasink.c
@@ -0,0 +1,137 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+Data sink interface.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#include
+#include "common.h"
+#include "datasink.h"
+#include "ds_compress.h"
+#include "ds_archive.h"
+#include "ds_xbstream.h"
+#include "ds_local.h"
+#include "ds_stdout.h"
+#include "ds_tmpfile.h"
+#include "ds_buffer.h"
+
+/************************************************************************
+Create a datasink of the specified type */
+ds_ctxt_t *
+ds_create(const char *root, ds_type_t type)
+{
+ datasink_t *ds;
+ ds_ctxt_t *ctxt;
+
+ switch (type) {
+ case DS_TYPE_STDOUT:
+ ds = &datasink_stdout;
+ break;
+ case DS_TYPE_LOCAL:
+ ds = &datasink_local;
+ break;
+ case DS_TYPE_ARCHIVE:
+#ifdef HAVE_LIBARCHIVE
+ ds = &datasink_archive;
+#else
+ msg("Error : mariabackup was built without libarchive support");
+ exit(EXIT_FAILURE);
+#endif
+ break;
+ case DS_TYPE_XBSTREAM:
+ ds = &datasink_xbstream;
+ break;
+ case DS_TYPE_COMPRESS:
+ ds = &datasink_compress;
+ break;
+ case DS_TYPE_ENCRYPT:
+ case DS_TYPE_DECRYPT:
+ msg("Error : mariabackup does not support encrypted backups.");
+ exit(EXIT_FAILURE);
+ break;
+
+ case DS_TYPE_TMPFILE:
+ ds = &datasink_tmpfile;
+ break;
+ case DS_TYPE_BUFFER:
+ ds = &datasink_buffer;
+ break;
+ default:
+ msg("Unknown datasink type: %d\n", type);
+ xb_ad(0);
+ return NULL;
+ }
+
+ ctxt = ds->init(root);
+ if (ctxt != NULL) {
+ ctxt->datasink = ds;
+ } else {
+ msg("Error: failed to initialize datasink.\n");
+ exit(EXIT_FAILURE);
+ }
+
+ return ctxt;
+}
+
+/************************************************************************
+Open a datasink file */
+ds_file_t *
+ds_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat)
+{
+ ds_file_t *file;
+
+ file = ctxt->datasink->open(ctxt, path, stat);
+ if (file != NULL) {
+ file->datasink = ctxt->datasink;
+ }
+
+ return file;
+}
+
+/************************************************************************
+Write to a datasink file.
+@return 0 on success, 1 on error. */
+int
+ds_write(ds_file_t *file, const void *buf, size_t len)
+{
+ return file->datasink->write(file, buf, len);
+}
+
+/************************************************************************
+Close a datasink file.
+@return 0 on success, 1, on error. */
+int
+ds_close(ds_file_t *file)
+{
+ return file->datasink->close(file);
+}
+
+/************************************************************************
+Destroy a datasink handle */
+void
+ds_destroy(ds_ctxt_t *ctxt)
+{
+ ctxt->datasink->deinit(ctxt);
+}
+
+/************************************************************************
+Set the destination pipe for a datasink (only makes sense for compress and
+tmpfile). */
+void ds_set_pipe(ds_ctxt_t *ctxt, ds_ctxt_t *pipe_ctxt)
+{
+ ctxt->pipe_ctxt = pipe_ctxt;
+}
diff --git a/extra/mariabackup/datasink.h b/extra/mariabackup/datasink.h
new file mode 100644
index 00000000000..8bf1321aad1
--- /dev/null
+++ b/extra/mariabackup/datasink.h
@@ -0,0 +1,100 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+Data sink interface.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#ifndef XB_DATASINK_H
+#define XB_DATASINK_H
+
+#include
+#include
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern char *xtrabackup_tmpdir;
+struct datasink_struct;
+typedef struct datasink_struct datasink_t;
+
+typedef struct ds_ctxt {
+ datasink_t *datasink;
+ char *root;
+ void *ptr;
+ struct ds_ctxt *pipe_ctxt;
+} ds_ctxt_t;
+
+typedef struct {
+ void *ptr;
+ char *path;
+ datasink_t *datasink;
+} ds_file_t;
+
+struct datasink_struct {
+ ds_ctxt_t *(*init)(const char *root);
+ ds_file_t *(*open)(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat);
+ int (*write)(ds_file_t *file, const void *buf, size_t len);
+ int (*close)(ds_file_t *file);
+ void (*deinit)(ds_ctxt_t *ctxt);
+};
+
+/* Supported datasink types */
+typedef enum {
+ DS_TYPE_STDOUT,
+ DS_TYPE_LOCAL,
+ DS_TYPE_ARCHIVE,
+ DS_TYPE_XBSTREAM,
+ DS_TYPE_COMPRESS,
+ DS_TYPE_ENCRYPT,
+ DS_TYPE_DECRYPT,
+ DS_TYPE_TMPFILE,
+ DS_TYPE_BUFFER
+} ds_type_t;
+
+/************************************************************************
+Create a datasink of the specified type */
+ds_ctxt_t *ds_create(const char *root, ds_type_t type);
+
+/************************************************************************
+Open a datasink file */
+ds_file_t *ds_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *stat);
+
+/************************************************************************
+Write to a datasink file.
+@return 0 on success, 1 on error. */
+int ds_write(ds_file_t *file, const void *buf, size_t len);
+
+/************************************************************************
+Close a datasink file.
+@return 0 on success, 1, on error. */
+int ds_close(ds_file_t *file);
+
+/************************************************************************
+Destroy a datasink handle */
+void ds_destroy(ds_ctxt_t *ctxt);
+
+/************************************************************************
+Set the destination pipe for a datasink (only makes sense for compress and
+tmpfile). */
+void ds_set_pipe(ds_ctxt_t *ctxt, ds_ctxt_t *pipe_ctxt);
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif /* XB_DATASINK_H */
diff --git a/extra/mariabackup/ds_archive.c b/extra/mariabackup/ds_archive.c
new file mode 100644
index 00000000000..50afcce4bc7
--- /dev/null
+++ b/extra/mariabackup/ds_archive.c
@@ -0,0 +1,280 @@
+/******************************************************
+Copyright (c) 2013 Percona LLC and/or its affiliates.
+
+Streaming implementation for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#include
+#include
+#include
+#include "common.h"
+#include "datasink.h"
+
+#if ARCHIVE_VERSION_NUMBER < 3000000
+#define archive_write_add_filter_none(X) archive_write_set_compression_none(X)
+#define archive_write_free(X) archive_write_finish(X)
+#endif
+
+typedef struct {
+ struct archive *archive;
+ ds_file_t *dest_file;
+ pthread_mutex_t mutex;
+} ds_archive_ctxt_t;
+
+typedef struct {
+ struct archive_entry *entry;
+ ds_archive_ctxt_t *archive_ctxt;
+} ds_archive_file_t;
+
+
+/***********************************************************************
+General archive interface */
+
+static ds_ctxt_t *archive_init(const char *root);
+static ds_file_t *archive_open(ds_ctxt_t *ctxt, const char *path,
+ MY_STAT *mystat);
+static int archive_write(ds_file_t *file, const void *buf, size_t len);
+static int archive_close(ds_file_t *file);
+static void archive_deinit(ds_ctxt_t *ctxt);
+
+datasink_t datasink_archive = {
+ &archive_init,
+ &archive_open,
+ &archive_write,
+ &archive_close,
+ &archive_deinit
+};
+
+static
+int
+my_archive_open_callback(struct archive *a __attribute__((unused)),
+ void *data __attribute__((unused)))
+{
+ return ARCHIVE_OK;
+}
+
+static
+ssize_t
+my_archive_write_callback(struct archive *a __attribute__((unused)),
+ void *data, const void *buffer, size_t length)
+{
+ ds_archive_ctxt_t *archive_ctxt;
+
+ archive_ctxt = (ds_archive_ctxt_t *) data;
+
+ xb_ad(archive_ctxt != NULL);
+ xb_ad(archive_ctxt->dest_file != NULL);
+
+ if (!ds_write(archive_ctxt->dest_file, buffer, length)) {
+ return length;
+ }
+ return -1;
+}
+
+static
+int
+my_archive_close_callback(struct archive *a __attribute__((unused)),
+ void *data __attribute__((unused)))
+{
+ return ARCHIVE_OK;
+}
+
+static
+ds_ctxt_t *
+archive_init(const char *root __attribute__((unused)))
+{
+ ds_ctxt_t *ctxt;
+ ds_archive_ctxt_t *archive_ctxt;
+ struct archive *a;
+
+ ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_archive_ctxt_t),
+ MYF(MY_FAE));
+ archive_ctxt = (ds_archive_ctxt_t *)(ctxt + 1);
+
+ if (pthread_mutex_init(&archive_ctxt->mutex, NULL)) {
+ msg("archive_init: pthread_mutex_init() failed.\n");
+ goto err;
+ }
+
+ a = archive_write_new();
+ if (a == NULL) {
+ msg("archive_write_new() failed.\n");
+ goto err;
+ }
+
+ archive_ctxt->archive = a;
+ archive_ctxt->dest_file = NULL;
+
+ if(archive_write_add_filter_none(a) != ARCHIVE_OK ||
+ archive_write_set_format_pax_restricted(a) != ARCHIVE_OK ||
+ /* disable internal buffering so we don't have to flush the
+ output in xtrabackup */
+ archive_write_set_bytes_per_block(a, 0) != ARCHIVE_OK) {
+ msg("failed to set libarchive archive options: %s\n",
+ archive_error_string(a));
+ archive_write_free(a);
+ goto err;
+ }
+
+ if (archive_write_open(a, archive_ctxt, my_archive_open_callback,
+ my_archive_write_callback,
+ my_archive_close_callback) != ARCHIVE_OK) {
+ msg("cannot open output archive.\n");
+ return NULL;
+ }
+
+ ctxt->ptr = archive_ctxt;
+
+ return ctxt;
+
+err:
+ my_free(ctxt);
+ return NULL;
+}
+
+static
+ds_file_t *
+archive_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
+{
+ ds_archive_ctxt_t *archive_ctxt;
+ ds_ctxt_t *dest_ctxt;
+ ds_file_t *file;
+ ds_archive_file_t *archive_file;
+
+ struct archive *a;
+ struct archive_entry *entry;
+
+ xb_ad(ctxt->pipe_ctxt != NULL);
+ dest_ctxt = ctxt->pipe_ctxt;
+
+ archive_ctxt = (ds_archive_ctxt_t *) ctxt->ptr;
+
+ pthread_mutex_lock(&archive_ctxt->mutex);
+ if (archive_ctxt->dest_file == NULL) {
+ archive_ctxt->dest_file = ds_open(dest_ctxt, path, mystat);
+ if (archive_ctxt->dest_file == NULL) {
+ return NULL;
+ }
+ }
+ pthread_mutex_unlock(&archive_ctxt->mutex);
+
+ file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
+ sizeof(ds_archive_file_t),
+ MYF(MY_FAE));
+
+ archive_file = (ds_archive_file_t *) (file + 1);
+
+ a = archive_ctxt->archive;
+
+ entry = archive_entry_new();
+ if (entry == NULL) {
+ msg("archive_entry_new() failed.\n");
+ goto err;
+ }
+
+ archive_entry_set_size(entry, mystat->st_size);
+ archive_entry_set_mode(entry, 0660);
+ archive_entry_set_filetype(entry, AE_IFREG);
+ archive_entry_set_pathname(entry, path);
+ archive_entry_set_mtime(entry, mystat->st_mtime, 0);
+
+ archive_file->entry = entry;
+ archive_file->archive_ctxt = archive_ctxt;
+
+ if (archive_write_header(a, entry) != ARCHIVE_OK) {
+ msg("archive_write_header() failed.\n");
+ archive_entry_free(entry);
+ goto err;
+ }
+
+ file->ptr = archive_file;
+ file->path = archive_ctxt->dest_file->path;
+
+ return file;
+
+err:
+ if (archive_ctxt->dest_file) {
+ ds_close(archive_ctxt->dest_file);
+ archive_ctxt->dest_file = NULL;
+ }
+ my_free(file);
+
+ return NULL;
+}
+
+static
+int
+archive_write(ds_file_t *file, const void *buf, size_t len)
+{
+ ds_archive_file_t *archive_file;
+ struct archive *a;
+
+ archive_file = (ds_archive_file_t *) file->ptr;
+
+ a = archive_file->archive_ctxt->archive;
+
+ xb_ad(archive_file->archive_ctxt->dest_file != NULL);
+ if (archive_write_data(a, buf, len) < 0) {
+ msg("archive_write_data() failed: %s (errno = %d)\n",
+ archive_error_string(a), archive_errno(a));
+ return 1;
+ }
+
+ return 0;
+}
+
+static
+int
+archive_close(ds_file_t *file)
+{
+ ds_archive_file_t *archive_file;
+ int rc = 0;
+
+ archive_file = (ds_archive_file_t *)file->ptr;
+
+ archive_entry_free(archive_file->entry);
+
+ my_free(file);
+
+ return rc;
+}
+
+static
+void
+archive_deinit(ds_ctxt_t *ctxt)
+{
+ struct archive *a;
+ ds_archive_ctxt_t *archive_ctxt;
+
+ archive_ctxt = (ds_archive_ctxt_t *) ctxt->ptr;
+
+ a = archive_ctxt->archive;
+
+ if (archive_write_close(a) != ARCHIVE_OK) {
+ msg("archive_write_close() failed.\n");
+ }
+ archive_write_free(a);
+
+ if (archive_ctxt->dest_file) {
+ ds_close(archive_ctxt->dest_file);
+ archive_ctxt->dest_file = NULL;
+ }
+
+ pthread_mutex_destroy(&archive_ctxt->mutex);
+
+ my_free(ctxt);
+}
diff --git a/extra/mariabackup/ds_archive.h b/extra/mariabackup/ds_archive.h
new file mode 100644
index 00000000000..3f4e4463c58
--- /dev/null
+++ b/extra/mariabackup/ds_archive.h
@@ -0,0 +1,28 @@
+/******************************************************
+Copyright (c) 2013 Percona LLC and/or its affiliates.
+
+Streaming interface for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#ifndef DS_ARCHIVE_H
+#define DS_ARCHIVE_H
+
+#include "datasink.h"
+
+extern datasink_t datasink_archive;
+
+#endif
diff --git a/extra/mariabackup/ds_buffer.c b/extra/mariabackup/ds_buffer.c
new file mode 100644
index 00000000000..4bb314c0f50
--- /dev/null
+++ b/extra/mariabackup/ds_buffer.c
@@ -0,0 +1,189 @@
+/******************************************************
+Copyright (c) 2012-2013 Percona LLC and/or its affiliates.
+
+buffer datasink for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+/* Does buffered output to a destination datasink set with ds_set_pipe().
+Writes to the destination datasink are guaranteed to not be smaller than a
+specified buffer size (DS_DEFAULT_BUFFER_SIZE by default), with the only
+exception for the last write for a file. */
+
+#include
+#include
+#include "ds_buffer.h"
+#include "common.h"
+#include "datasink.h"
+
+#define DS_DEFAULT_BUFFER_SIZE (64 * 1024)
+
+typedef struct {
+ ds_file_t *dst_file;
+ char *buf;
+ size_t pos;
+ size_t size;
+} ds_buffer_file_t;
+
+typedef struct {
+ size_t buffer_size;
+} ds_buffer_ctxt_t;
+
+static ds_ctxt_t *buffer_init(const char *root);
+static ds_file_t *buffer_open(ds_ctxt_t *ctxt, const char *path,
+ MY_STAT *mystat);
+static int buffer_write(ds_file_t *file, const void *buf, size_t len);
+static int buffer_close(ds_file_t *file);
+static void buffer_deinit(ds_ctxt_t *ctxt);
+
+datasink_t datasink_buffer = {
+ &buffer_init,
+ &buffer_open,
+ &buffer_write,
+ &buffer_close,
+ &buffer_deinit
+};
+
+/* Change the default buffer size */
+void ds_buffer_set_size(ds_ctxt_t *ctxt, size_t size)
+{
+ ds_buffer_ctxt_t *buffer_ctxt = (ds_buffer_ctxt_t *) ctxt->ptr;
+
+ buffer_ctxt->buffer_size = size;
+}
+
+static ds_ctxt_t *
+buffer_init(const char *root)
+{
+ ds_ctxt_t *ctxt;
+ ds_buffer_ctxt_t *buffer_ctxt;
+
+ ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_buffer_ctxt_t),
+ MYF(MY_FAE));
+ buffer_ctxt = (ds_buffer_ctxt_t *) (ctxt + 1);
+ buffer_ctxt->buffer_size = DS_DEFAULT_BUFFER_SIZE;
+
+ ctxt->ptr = buffer_ctxt;
+ ctxt->root = my_strdup(root, MYF(MY_FAE));
+
+ return ctxt;
+}
+
+static ds_file_t *
+buffer_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
+{
+ ds_buffer_ctxt_t *buffer_ctxt;
+ ds_ctxt_t *pipe_ctxt;
+ ds_file_t *dst_file;
+ ds_file_t *file;
+ ds_buffer_file_t *buffer_file;
+
+ pipe_ctxt = ctxt->pipe_ctxt;
+ xb_a(pipe_ctxt != NULL);
+
+ dst_file = ds_open(pipe_ctxt, path, mystat);
+ if (dst_file == NULL) {
+ exit(EXIT_FAILURE);
+ }
+
+ buffer_ctxt = (ds_buffer_ctxt_t *) ctxt->ptr;
+
+ file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
+ sizeof(ds_buffer_file_t) +
+ buffer_ctxt->buffer_size,
+ MYF(MY_FAE));
+
+ buffer_file = (ds_buffer_file_t *) (file + 1);
+ buffer_file->dst_file = dst_file;
+ buffer_file->buf = (char *) (buffer_file + 1);
+ buffer_file->size = buffer_ctxt->buffer_size;
+ buffer_file->pos = 0;
+
+ file->path = dst_file->path;
+ file->ptr = buffer_file;
+
+ return file;
+}
+
+static int
+buffer_write(ds_file_t *file, const void *buf, size_t len)
+{
+ ds_buffer_file_t *buffer_file;
+
+ buffer_file = (ds_buffer_file_t *) file->ptr;
+
+ while (len > 0) {
+ if (buffer_file->pos + len > buffer_file->size) {
+ if (buffer_file->pos > 0) {
+ size_t bytes;
+
+ bytes = buffer_file->size - buffer_file->pos;
+ memcpy(buffer_file->buf + buffer_file->pos, buf,
+ bytes);
+
+ if (ds_write(buffer_file->dst_file,
+ buffer_file->buf,
+ buffer_file->size)) {
+ return 1;
+ }
+
+ buffer_file->pos = 0;
+
+ buf = (const char *) buf + bytes;
+ len -= bytes;
+ } else {
+ /* We don't have any buffered bytes, just write
+ the entire source buffer */
+ if (ds_write(buffer_file->dst_file, buf, len)) {
+ return 1;
+ }
+ break;
+ }
+ } else {
+ memcpy(buffer_file->buf + buffer_file->pos, buf, len);
+ buffer_file->pos += len;
+ break;
+ }
+ }
+
+ return 0;
+}
+
+static int
+buffer_close(ds_file_t *file)
+{
+ ds_buffer_file_t *buffer_file;
+ int ret;
+
+ buffer_file = (ds_buffer_file_t *) file->ptr;
+ if (buffer_file->pos > 0) {
+ ds_write(buffer_file->dst_file, buffer_file->buf,
+ buffer_file->pos);
+ }
+
+ ret = ds_close(buffer_file->dst_file);
+
+ my_free(file);
+
+ return ret;
+}
+
+static void
+buffer_deinit(ds_ctxt_t *ctxt)
+{
+ my_free(ctxt->root);
+ my_free(ctxt);
+}
diff --git a/extra/mariabackup/ds_buffer.h b/extra/mariabackup/ds_buffer.h
new file mode 100644
index 00000000000..f8d2d63267d
--- /dev/null
+++ b/extra/mariabackup/ds_buffer.h
@@ -0,0 +1,39 @@
+/******************************************************
+Copyright (c) 2012-2013 Percona LLC and/or its affiliates.
+
+buffer datasink for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#ifndef DS_BUFFER_H
+#define DS_BUFFER_H
+
+#include "datasink.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern datasink_t datasink_buffer;
+
+/* Change the default buffer size */
+void ds_buffer_set_size(ds_ctxt_t *ctxt, size_t size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/extra/mariabackup/ds_compress.c b/extra/mariabackup/ds_compress.c
new file mode 100644
index 00000000000..15801c8abd4
--- /dev/null
+++ b/extra/mariabackup/ds_compress.c
@@ -0,0 +1,462 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+Compressing datasink implementation for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#include
+#include
+#include
+#include
+#include "common.h"
+#include "datasink.h"
+
+#define COMPRESS_CHUNK_SIZE ((size_t) (xtrabackup_compress_chunk_size))
+#define MY_QLZ_COMPRESS_OVERHEAD 400
+
+typedef struct {
+ pthread_t id;
+ uint num;
+ pthread_mutex_t ctrl_mutex;
+ pthread_cond_t ctrl_cond;
+ pthread_mutex_t data_mutex;
+ pthread_cond_t data_cond;
+ my_bool started;
+ my_bool data_avail;
+ my_bool cancelled;
+ const char *from;
+ size_t from_len;
+ char *to;
+ size_t to_len;
+ qlz_state_compress state;
+ ulong adler;
+} comp_thread_ctxt_t;
+
+typedef struct {
+ comp_thread_ctxt_t *threads;
+ uint nthreads;
+} ds_compress_ctxt_t;
+
+typedef struct {
+ ds_file_t *dest_file;
+ ds_compress_ctxt_t *comp_ctxt;
+ size_t bytes_processed;
+} ds_compress_file_t;
+
+/* Compression options */
+extern char *xtrabackup_compress_alg;
+extern uint xtrabackup_compress_threads;
+extern ulonglong xtrabackup_compress_chunk_size;
+
+static ds_ctxt_t *compress_init(const char *root);
+static ds_file_t *compress_open(ds_ctxt_t *ctxt, const char *path,
+ MY_STAT *mystat);
+static int compress_write(ds_file_t *file, const void *buf, size_t len);
+static int compress_close(ds_file_t *file);
+static void compress_deinit(ds_ctxt_t *ctxt);
+
+datasink_t datasink_compress = {
+ &compress_init,
+ &compress_open,
+ &compress_write,
+ &compress_close,
+ &compress_deinit
+};
+
+static inline int write_uint32_le(ds_file_t *file, ulong n);
+static inline int write_uint64_le(ds_file_t *file, ulonglong n);
+
+static comp_thread_ctxt_t *create_worker_threads(uint n);
+static void destroy_worker_threads(comp_thread_ctxt_t *threads, uint n);
+static void *compress_worker_thread_func(void *arg);
+
+static
+ds_ctxt_t *
+compress_init(const char *root)
+{
+ ds_ctxt_t *ctxt;
+ ds_compress_ctxt_t *compress_ctxt;
+ comp_thread_ctxt_t *threads;
+
+ /* Create and initialize the worker threads */
+ threads = create_worker_threads(xtrabackup_compress_threads);
+ if (threads == NULL) {
+ msg("compress: failed to create worker threads.\n");
+ return NULL;
+ }
+
+ ctxt = (ds_ctxt_t *) my_malloc(sizeof(ds_ctxt_t) +
+ sizeof(ds_compress_ctxt_t),
+ MYF(MY_FAE));
+
+ compress_ctxt = (ds_compress_ctxt_t *) (ctxt + 1);
+ compress_ctxt->threads = threads;
+ compress_ctxt->nthreads = xtrabackup_compress_threads;
+
+ ctxt->ptr = compress_ctxt;
+ ctxt->root = my_strdup(root, MYF(MY_FAE));
+
+ return ctxt;
+}
+
+static
+ds_file_t *
+compress_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
+{
+ ds_compress_ctxt_t *comp_ctxt;
+ ds_ctxt_t *dest_ctxt;
+ ds_file_t *dest_file;
+ char new_name[FN_REFLEN];
+ size_t name_len;
+ ds_file_t *file;
+ ds_compress_file_t *comp_file;
+
+ xb_ad(ctxt->pipe_ctxt != NULL);
+ dest_ctxt = ctxt->pipe_ctxt;
+
+ comp_ctxt = (ds_compress_ctxt_t *) ctxt->ptr;
+
+ /* Append the .qp extension to the filename */
+ fn_format(new_name, path, "", ".qp", MYF(MY_APPEND_EXT));
+
+ dest_file = ds_open(dest_ctxt, new_name, mystat);
+ if (dest_file == NULL) {
+ return NULL;
+ }
+
+ /* Write the qpress archive header */
+ if (ds_write(dest_file, "qpress10", 8) ||
+ write_uint64_le(dest_file, COMPRESS_CHUNK_SIZE)) {
+ goto err;
+ }
+
+ /* We are going to create a one-file "flat" (i.e. with no
+ subdirectories) archive. So strip the directory part from the path and
+ remove the '.qp' suffix. */
+ fn_format(new_name, path, "", "", MYF(MY_REPLACE_DIR));
+
+ /* Write the qpress file header */
+ name_len = strlen(new_name);
+ if (ds_write(dest_file, "F", 1) ||
+ write_uint32_le(dest_file, (uint)name_len) ||
+ /* we want to write the terminating \0 as well */
+ ds_write(dest_file, new_name, name_len + 1)) {
+ goto err;
+ }
+
+ file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
+ sizeof(ds_compress_file_t),
+ MYF(MY_FAE));
+ comp_file = (ds_compress_file_t *) (file + 1);
+ comp_file->dest_file = dest_file;
+ comp_file->comp_ctxt = comp_ctxt;
+ comp_file->bytes_processed = 0;
+
+ file->ptr = comp_file;
+ file->path = dest_file->path;
+
+ return file;
+
+err:
+ ds_close(dest_file);
+ return NULL;
+}
+
+static
+int
+compress_write(ds_file_t *file, const void *buf, size_t len)
+{
+ ds_compress_file_t *comp_file;
+ ds_compress_ctxt_t *comp_ctxt;
+ comp_thread_ctxt_t *threads;
+ comp_thread_ctxt_t *thd;
+ uint nthreads;
+ uint i;
+ const char *ptr;
+ ds_file_t *dest_file;
+
+ comp_file = (ds_compress_file_t *) file->ptr;
+ comp_ctxt = comp_file->comp_ctxt;
+ dest_file = comp_file->dest_file;
+
+ threads = comp_ctxt->threads;
+ nthreads = comp_ctxt->nthreads;
+
+ ptr = (const char *) buf;
+ while (len > 0) {
+ uint max_thread;
+
+ /* Send data to worker threads for compression */
+ for (i = 0; i < nthreads; i++) {
+ size_t chunk_len;
+
+ thd = threads + i;
+
+ pthread_mutex_lock(&thd->ctrl_mutex);
+
+ chunk_len = (len > COMPRESS_CHUNK_SIZE) ?
+ COMPRESS_CHUNK_SIZE : len;
+ thd->from = ptr;
+ thd->from_len = chunk_len;
+
+ pthread_mutex_lock(&thd->data_mutex);
+ thd->data_avail = TRUE;
+ pthread_cond_signal(&thd->data_cond);
+ pthread_mutex_unlock(&thd->data_mutex);
+
+ len -= chunk_len;
+ if (len == 0) {
+ break;
+ }
+ ptr += chunk_len;
+ }
+
+ max_thread = (i < nthreads) ? i : nthreads - 1;
+
+ /* Reap and stream the compressed data */
+ for (i = 0; i <= max_thread; i++) {
+ thd = threads + i;
+
+ pthread_mutex_lock(&thd->data_mutex);
+ while (thd->data_avail == TRUE) {
+ pthread_cond_wait(&thd->data_cond,
+ &thd->data_mutex);
+ }
+
+ xb_a(threads[i].to_len > 0);
+
+ if (ds_write(dest_file, "NEWBNEWB", 8) ||
+ write_uint64_le(dest_file,
+ comp_file->bytes_processed)) {
+ msg("compress: write to the destination stream "
+ "failed.\n");
+ return 1;
+ }
+
+ comp_file->bytes_processed += threads[i].from_len;
+
+ if (write_uint32_le(dest_file, threads[i].adler) ||
+ ds_write(dest_file, threads[i].to,
+ threads[i].to_len)) {
+ msg("compress: write to the destination stream "
+ "failed.\n");
+ return 1;
+ }
+
+ pthread_mutex_unlock(&threads[i].data_mutex);
+ pthread_mutex_unlock(&threads[i].ctrl_mutex);
+ }
+ }
+
+ return 0;
+}
+
+static
+int
+compress_close(ds_file_t *file)
+{
+ ds_compress_file_t *comp_file;
+ ds_file_t *dest_file;
+ int rc;
+
+ comp_file = (ds_compress_file_t *) file->ptr;
+ dest_file = comp_file->dest_file;
+
+ /* Write the qpress file trailer */
+ ds_write(dest_file, "ENDSENDS", 8);
+
+ /* Supposedly the number of written bytes should be written as a
+ "recovery information" in the file trailer, but in reality qpress
+ always writes 8 zeros here. Let's do the same */
+
+ write_uint64_le(dest_file, 0);
+
+ rc = ds_close(dest_file);
+
+ my_free(file);
+
+ return rc;
+}
+
+static
+void
+compress_deinit(ds_ctxt_t *ctxt)
+{
+ ds_compress_ctxt_t *comp_ctxt;
+
+ xb_ad(ctxt->pipe_ctxt != NULL);
+
+ comp_ctxt = (ds_compress_ctxt_t *) ctxt->ptr;;
+
+ destroy_worker_threads(comp_ctxt->threads, comp_ctxt->nthreads);
+
+ my_free(ctxt->root);
+ my_free(ctxt);
+}
+
+static inline
+int
+write_uint32_le(ds_file_t *file, ulong n)
+{
+ char tmp[4];
+
+ int4store(tmp, n);
+ return ds_write(file, tmp, sizeof(tmp));
+}
+
+static inline
+int
+write_uint64_le(ds_file_t *file, ulonglong n)
+{
+ char tmp[8];
+
+ int8store(tmp, n);
+ return ds_write(file, tmp, sizeof(tmp));
+}
+
+static
+comp_thread_ctxt_t *
+create_worker_threads(uint n)
+{
+ comp_thread_ctxt_t *threads;
+ uint i;
+
+ threads = (comp_thread_ctxt_t *)
+ my_malloc(sizeof(comp_thread_ctxt_t) * n, MYF(MY_FAE));
+
+ for (i = 0; i < n; i++) {
+ comp_thread_ctxt_t *thd = threads + i;
+
+ thd->num = i + 1;
+ thd->started = FALSE;
+ thd->cancelled = FALSE;
+ thd->data_avail = FALSE;
+
+ thd->to = (char *) my_malloc(COMPRESS_CHUNK_SIZE +
+ MY_QLZ_COMPRESS_OVERHEAD,
+ MYF(MY_FAE));
+
+ /* Initialize the control mutex and condition var */
+ if (pthread_mutex_init(&thd->ctrl_mutex, NULL) ||
+ pthread_cond_init(&thd->ctrl_cond, NULL)) {
+ goto err;
+ }
+
+ /* Initialize and data mutex and condition var */
+ if (pthread_mutex_init(&thd->data_mutex, NULL) ||
+ pthread_cond_init(&thd->data_cond, NULL)) {
+ goto err;
+ }
+
+ pthread_mutex_lock(&thd->ctrl_mutex);
+
+ if (pthread_create(&thd->id, NULL, compress_worker_thread_func,
+ thd)) {
+ msg("compress: pthread_create() failed: "
+ "errno = %d\n", errno);
+ goto err;
+ }
+ }
+
+ /* Wait for the threads to start */
+ for (i = 0; i < n; i++) {
+ comp_thread_ctxt_t *thd = threads + i;
+
+ while (thd->started == FALSE)
+ pthread_cond_wait(&thd->ctrl_cond, &thd->ctrl_mutex);
+ pthread_mutex_unlock(&thd->ctrl_mutex);
+ }
+
+ return threads;
+
+err:
+ return NULL;
+}
+
+static
+void
+destroy_worker_threads(comp_thread_ctxt_t *threads, uint n)
+{
+ uint i;
+
+ for (i = 0; i < n; i++) {
+ comp_thread_ctxt_t *thd = threads + i;
+
+ pthread_mutex_lock(&thd->data_mutex);
+ threads[i].cancelled = TRUE;
+ pthread_cond_signal(&thd->data_cond);
+ pthread_mutex_unlock(&thd->data_mutex);
+
+ pthread_join(thd->id, NULL);
+
+ pthread_cond_destroy(&thd->data_cond);
+ pthread_mutex_destroy(&thd->data_mutex);
+ pthread_cond_destroy(&thd->ctrl_cond);
+ pthread_mutex_destroy(&thd->ctrl_mutex);
+
+ my_free(thd->to);
+ }
+
+ my_free(threads);
+}
+
+static
+void *
+compress_worker_thread_func(void *arg)
+{
+ comp_thread_ctxt_t *thd = (comp_thread_ctxt_t *) arg;
+
+ pthread_mutex_lock(&thd->ctrl_mutex);
+
+ pthread_mutex_lock(&thd->data_mutex);
+
+ thd->started = TRUE;
+ pthread_cond_signal(&thd->ctrl_cond);
+
+ pthread_mutex_unlock(&thd->ctrl_mutex);
+
+ while (1) {
+ thd->data_avail = FALSE;
+ pthread_cond_signal(&thd->data_cond);
+
+ while (!thd->data_avail && !thd->cancelled) {
+ pthread_cond_wait(&thd->data_cond, &thd->data_mutex);
+ }
+
+ if (thd->cancelled)
+ break;
+
+ thd->to_len = qlz_compress(thd->from, thd->to, thd->from_len,
+ &thd->state);
+
+ /* qpress uses 0x00010000 as the initial value, but its own
+ Adler-32 implementation treats the value differently:
+ 1. higher order bits are the sum of all bytes in the sequence
+ 2. lower order bits are the sum of resulting values at every
+ step.
+ So it's the other way around as compared to zlib's adler32().
+ That's why 0x00000001 is being passed here to be compatible
+ with qpress implementation. */
+
+ thd->adler = adler32(0x00000001, (uchar *) thd->to,
+ (uInt)thd->to_len);
+ }
+
+ pthread_mutex_unlock(&thd->data_mutex);
+
+ return NULL;
+}
diff --git a/extra/mariabackup/ds_compress.h b/extra/mariabackup/ds_compress.h
new file mode 100644
index 00000000000..8498c965e13
--- /dev/null
+++ b/extra/mariabackup/ds_compress.h
@@ -0,0 +1,28 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+Compression interface for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#ifndef DS_COMPRESS_H
+#define DS_COMPRESS_H
+
+#include "datasink.h"
+
+extern datasink_t datasink_compress;
+
+#endif
diff --git a/extra/mariabackup/ds_local.c b/extra/mariabackup/ds_local.c
new file mode 100644
index 00000000000..3e2b1e0129b
--- /dev/null
+++ b/extra/mariabackup/ds_local.c
@@ -0,0 +1,151 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+Local datasink implementation for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#include
+#include
+#include
+#include "common.h"
+#include "datasink.h"
+
+typedef struct {
+ File fd;
+} ds_local_file_t;
+
+static ds_ctxt_t *local_init(const char *root);
+static ds_file_t *local_open(ds_ctxt_t *ctxt, const char *path,
+ MY_STAT *mystat);
+static int local_write(ds_file_t *file, const void *buf, size_t len);
+static int local_close(ds_file_t *file);
+static void local_deinit(ds_ctxt_t *ctxt);
+
+datasink_t datasink_local = {
+ &local_init,
+ &local_open,
+ &local_write,
+ &local_close,
+ &local_deinit
+};
+
+static
+ds_ctxt_t *
+local_init(const char *root)
+{
+ ds_ctxt_t *ctxt;
+
+ if (my_mkdir(root, 0777, MYF(0)) < 0
+ && my_errno != EEXIST && my_errno != EISDIR)
+ {
+ char errbuf[MYSYS_STRERROR_SIZE];
+ my_strerror(errbuf, sizeof(errbuf),my_errno);
+ my_error(EE_CANT_MKDIR, MYF(ME_BELL | ME_WAITTANG),
+ root, my_errno,errbuf, my_errno);
+ return NULL;
+ }
+
+ ctxt = my_malloc(sizeof(ds_ctxt_t), MYF(MY_FAE));
+
+ ctxt->root = my_strdup(root, MYF(MY_FAE));
+
+ return ctxt;
+}
+
+static
+ds_file_t *
+local_open(ds_ctxt_t *ctxt, const char *path,
+ MY_STAT *mystat __attribute__((unused)))
+{
+ char fullpath[FN_REFLEN];
+ char dirpath[FN_REFLEN];
+ size_t dirpath_len;
+ size_t path_len;
+ ds_local_file_t *local_file;
+ ds_file_t *file;
+ File fd;
+
+ fn_format(fullpath, path, ctxt->root, "", MYF(MY_RELATIVE_PATH));
+
+ /* Create the directory if needed */
+ dirname_part(dirpath, fullpath, &dirpath_len);
+ if (my_mkdir(dirpath, 0777, MYF(0)) < 0 && my_errno != EEXIST) {
+ char errbuf[MYSYS_STRERROR_SIZE];
+ my_strerror(errbuf, sizeof(errbuf), my_errno);
+ my_error(EE_CANT_MKDIR, MYF(ME_BELL | ME_WAITTANG),
+ dirpath, my_errno, errbuf);
+ return NULL;
+ }
+
+ fd = my_create(fullpath, 0, O_WRONLY | O_BINARY | O_EXCL | O_NOFOLLOW,
+ MYF(MY_WME));
+ if (fd < 0) {
+ return NULL;
+ }
+
+ path_len = strlen(fullpath) + 1; /* terminating '\0' */
+
+ file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
+ sizeof(ds_local_file_t) +
+ path_len,
+ MYF(MY_FAE));
+ local_file = (ds_local_file_t *) (file + 1);
+
+ local_file->fd = fd;
+
+ file->path = (char *) local_file + sizeof(ds_local_file_t);
+ memcpy(file->path, fullpath, path_len);
+
+ file->ptr = local_file;
+
+ return file;
+}
+
+static
+int
+local_write(ds_file_t *file, const void *buf, size_t len)
+{
+ File fd = ((ds_local_file_t *) file->ptr)->fd;
+
+ if (!my_write(fd, buf, len, MYF(MY_WME | MY_NABP))) {
+ posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
+ return 0;
+ }
+
+ return 1;
+}
+
+static
+int
+local_close(ds_file_t *file)
+{
+ File fd = ((ds_local_file_t *) file->ptr)->fd;
+
+ my_free(file);
+
+ my_sync(fd, MYF(MY_WME));
+
+ return my_close(fd, MYF(MY_WME));
+}
+
+static
+void
+local_deinit(ds_ctxt_t *ctxt)
+{
+ my_free(ctxt->root);
+ my_free(ctxt);
+}
diff --git a/extra/mariabackup/ds_local.h b/extra/mariabackup/ds_local.h
new file mode 100644
index 00000000000..b0f0f04030c
--- /dev/null
+++ b/extra/mariabackup/ds_local.h
@@ -0,0 +1,28 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+Local datasink interface for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#ifndef DS_LOCAL_H
+#define DS_LOCAL_H
+
+#include "datasink.h"
+
+extern datasink_t datasink_local;
+
+#endif
diff --git a/extra/mariabackup/ds_stdout.c b/extra/mariabackup/ds_stdout.c
new file mode 100644
index 00000000000..91a514ddf64
--- /dev/null
+++ b/extra/mariabackup/ds_stdout.c
@@ -0,0 +1,121 @@
+/******************************************************
+Copyright (c) 2013 Percona LLC and/or its affiliates.
+
+Local datasink implementation for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#include
+#include
+#include "common.h"
+#include "datasink.h"
+
+typedef struct {
+ File fd;
+} ds_stdout_file_t;
+
+static ds_ctxt_t *stdout_init(const char *root);
+static ds_file_t *stdout_open(ds_ctxt_t *ctxt, const char *path,
+ MY_STAT *mystat);
+static int stdout_write(ds_file_t *file, const void *buf, size_t len);
+static int stdout_close(ds_file_t *file);
+static void stdout_deinit(ds_ctxt_t *ctxt);
+
+datasink_t datasink_stdout = {
+ &stdout_init,
+ &stdout_open,
+ &stdout_write,
+ &stdout_close,
+ &stdout_deinit
+};
+
+static
+ds_ctxt_t *
+stdout_init(const char *root)
+{
+ ds_ctxt_t *ctxt;
+
+ ctxt = my_malloc(sizeof(ds_ctxt_t), MYF(MY_FAE));
+
+ ctxt->root = my_strdup(root, MYF(MY_FAE));
+
+ return ctxt;
+}
+
+static
+ds_file_t *
+stdout_open(ds_ctxt_t *ctxt __attribute__((unused)),
+ const char *path __attribute__((unused)),
+ MY_STAT *mystat __attribute__((unused)))
+{
+ ds_stdout_file_t *stdout_file;
+ ds_file_t *file;
+ size_t pathlen;
+ const char *fullpath = "";
+
+ pathlen = strlen(fullpath) + 1;
+
+ file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
+ sizeof(ds_stdout_file_t) +
+ pathlen,
+ MYF(MY_FAE));
+ stdout_file = (ds_stdout_file_t *) (file + 1);
+
+
+#ifdef __WIN__
+ setmode(fileno(stdout), _O_BINARY);
+#endif
+
+ stdout_file->fd = my_fileno(stdout);
+
+ file->path = (char *) stdout_file + sizeof(ds_stdout_file_t);
+ memcpy(file->path, fullpath, pathlen);
+
+ file->ptr = stdout_file;
+
+ return file;
+}
+
+static
+int
+stdout_write(ds_file_t *file, const void *buf, size_t len)
+{
+ File fd = ((ds_stdout_file_t *) file->ptr)->fd;
+
+ if (!my_write(fd, buf, len, MYF(MY_WME | MY_NABP))) {
+ posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
+ return 0;
+ }
+
+ return 1;
+}
+
+static
+int
+stdout_close(ds_file_t *file)
+{
+ my_free(file);
+
+ return 1;
+}
+
+static
+void
+stdout_deinit(ds_ctxt_t *ctxt)
+{
+ my_free(ctxt->root);
+ my_free(ctxt);
+}
diff --git a/extra/mariabackup/ds_stdout.h b/extra/mariabackup/ds_stdout.h
new file mode 100644
index 00000000000..58940264fef
--- /dev/null
+++ b/extra/mariabackup/ds_stdout.h
@@ -0,0 +1,28 @@
+/******************************************************
+Copyright (c) 2013 Percona LLC and/or its affiliates.
+
+Local datasink interface for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#ifndef DS_STDOUT_H
+#define DS_STDOUT_H
+
+#include "datasink.h"
+
+extern datasink_t datasink_stdout;
+
+#endif
diff --git a/extra/mariabackup/ds_tmpfile.c b/extra/mariabackup/ds_tmpfile.c
new file mode 100644
index 00000000000..b039d83ba03
--- /dev/null
+++ b/extra/mariabackup/ds_tmpfile.c
@@ -0,0 +1,247 @@
+/******************************************************
+Copyright (c) 2012 Percona LLC and/or its affiliates.
+
+tmpfile datasink for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+/* Do all writes to temporary files first, then pipe them to the specified
+datasink in a serialized way in deinit(). */
+
+#include
+#include "common.h"
+#include "datasink.h"
+
+typedef struct {
+ pthread_mutex_t mutex;
+ LIST *file_list;
+} ds_tmpfile_ctxt_t;
+
+typedef struct {
+ LIST list;
+ File fd;
+ char *orig_path;
+ MY_STAT mystat;
+ ds_file_t *file;
+} ds_tmp_file_t;
+
+static ds_ctxt_t *tmpfile_init(const char *root);
+static ds_file_t *tmpfile_open(ds_ctxt_t *ctxt, const char *path,
+ MY_STAT *mystat);
+static int tmpfile_write(ds_file_t *file, const void *buf, size_t len);
+static int tmpfile_close(ds_file_t *file);
+static void tmpfile_deinit(ds_ctxt_t *ctxt);
+
+datasink_t datasink_tmpfile = {
+ &tmpfile_init,
+ &tmpfile_open,
+ &tmpfile_write,
+ &tmpfile_close,
+ &tmpfile_deinit
+};
+
+
+static ds_ctxt_t *
+tmpfile_init(const char *root)
+{
+ ds_ctxt_t *ctxt;
+ ds_tmpfile_ctxt_t *tmpfile_ctxt;
+
+ ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_tmpfile_ctxt_t),
+ MYF(MY_FAE));
+ tmpfile_ctxt = (ds_tmpfile_ctxt_t *) (ctxt + 1);
+ tmpfile_ctxt->file_list = NULL;
+ if (pthread_mutex_init(&tmpfile_ctxt->mutex, NULL)) {
+
+ my_free(ctxt);
+ return NULL;
+ }
+
+ ctxt->ptr = tmpfile_ctxt;
+ ctxt->root = my_strdup(root, MYF(MY_FAE));
+
+ return ctxt;
+}
+
+static ds_file_t *
+tmpfile_open(ds_ctxt_t *ctxt, const char *path,
+ MY_STAT *mystat)
+{
+ ds_tmpfile_ctxt_t *tmpfile_ctxt;
+ char tmp_path[FN_REFLEN];
+ ds_tmp_file_t *tmp_file;
+ ds_file_t *file;
+ size_t path_len;
+ File fd;
+
+ /* Create a temporary file in tmpdir. The file will be automatically
+ removed on close. Code copied from mysql_tmpfile(). */
+ fd = create_temp_file(tmp_path,xtrabackup_tmpdir,
+ "xbtemp",
+#ifdef __WIN__
+ O_BINARY | O_TRUNC | O_SEQUENTIAL |
+ O_TEMPORARY | O_SHORT_LIVED |
+#endif /* __WIN__ */
+ O_CREAT | O_EXCL | O_RDWR,
+ MYF(MY_WME));
+
+#ifndef __WIN__
+ if (fd >= 0) {
+ /* On Windows, open files cannot be removed, but files can be
+ created with the O_TEMPORARY flag to the same effect
+ ("delete on close"). */
+ unlink(tmp_path);
+ }
+#endif /* !__WIN__ */
+
+ if (fd < 0) {
+ return NULL;
+ }
+
+ path_len = strlen(path) + 1; /* terminating '\0' */
+
+ file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
+ sizeof(ds_tmp_file_t) + path_len,
+ MYF(MY_FAE));
+
+ tmp_file = (ds_tmp_file_t *) (file + 1);
+ tmp_file->file = file;
+ memcpy(&tmp_file->mystat, mystat, sizeof(MY_STAT));
+ /* Save a copy of 'path', since it may not be accessible later */
+ tmp_file->orig_path = (char *) tmp_file + sizeof(ds_tmp_file_t);
+
+ tmp_file->fd = fd;
+ memcpy(tmp_file->orig_path, path, path_len);
+
+ /* Store the real temporary file name in file->path */
+ file->path = my_strdup(tmp_path, MYF(MY_FAE));
+ file->ptr = tmp_file;
+
+ /* Store the file object in the list to be piped later */
+ tmpfile_ctxt = (ds_tmpfile_ctxt_t *) ctxt->ptr;
+ tmp_file->list.data = tmp_file;
+
+ pthread_mutex_lock(&tmpfile_ctxt->mutex);
+ tmpfile_ctxt->file_list = list_add(tmpfile_ctxt->file_list,
+ &tmp_file->list);
+ pthread_mutex_unlock(&tmpfile_ctxt->mutex);
+
+ return file;
+}
+
+static int
+tmpfile_write(ds_file_t *file, const void *buf, size_t len)
+{
+ File fd = ((ds_tmp_file_t *) file->ptr)->fd;
+
+ if (!my_write(fd, buf, len, MYF(MY_WME | MY_NABP))) {
+ posix_fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
+ return 0;
+ }
+
+ return 1;
+}
+
+static int
+tmpfile_close(ds_file_t *file)
+{
+ /* Do nothing -- we will close (and thus remove) the file after piping
+ it to the destination datasink in tmpfile_deinit(). */
+
+ my_free(file->path);
+
+ return 0;
+}
+
+static void
+tmpfile_deinit(ds_ctxt_t *ctxt)
+{
+ LIST *list;
+ ds_tmpfile_ctxt_t *tmpfile_ctxt;
+ MY_STAT mystat;
+ ds_tmp_file_t *tmp_file;
+ ds_file_t *dst_file;
+ ds_ctxt_t *pipe_ctxt;
+ void *buf = NULL;
+ const size_t buf_size = 10 * 1024 * 1024;
+ size_t bytes;
+ size_t offset;
+
+ pipe_ctxt = ctxt->pipe_ctxt;
+ xb_a(pipe_ctxt != NULL);
+
+ buf = my_malloc(buf_size, MYF(MY_FAE));
+
+ tmpfile_ctxt = (ds_tmpfile_ctxt_t *) ctxt->ptr;
+ list = tmpfile_ctxt->file_list;
+
+ /* Walk the files in the order they have been added */
+ list = list_reverse(list);
+ while (list != NULL) {
+ tmp_file = list->data;
+ /* Stat the file to replace size and mtime on the original
+ * mystat struct */
+ if (my_fstat(tmp_file->fd, &mystat, MYF(0))) {
+ msg("error: my_fstat() failed.\n");
+ exit(EXIT_FAILURE);
+ }
+ tmp_file->mystat.st_size = mystat.st_size;
+ tmp_file->mystat.st_mtime = mystat.st_mtime;
+
+ dst_file = ds_open(pipe_ctxt, tmp_file->orig_path,
+ &tmp_file->mystat);
+ if (dst_file == NULL) {
+ msg("error: could not stream a temporary file to "
+ "'%s'\n", tmp_file->orig_path);
+ exit(EXIT_FAILURE);
+ }
+
+ /* copy to the destination datasink */
+ posix_fadvise(tmp_file->fd, 0, 0, POSIX_FADV_SEQUENTIAL);
+ if (my_seek(tmp_file->fd, 0, SEEK_SET, MYF(0)) ==
+ MY_FILEPOS_ERROR) {
+ msg("error: my_seek() failed for '%s', errno = %d.\n",
+ tmp_file->file->path, my_errno);
+ exit(EXIT_FAILURE);
+ }
+ offset = 0;
+ while ((bytes = my_read(tmp_file->fd, buf, buf_size,
+ MYF(MY_WME))) > 0) {
+ posix_fadvise(tmp_file->fd, offset, buf_size, POSIX_FADV_DONTNEED);
+ offset += buf_size;
+ if (ds_write(dst_file, buf, bytes)) {
+ msg("error: cannot write to stream for '%s'.\n",
+ tmp_file->orig_path);
+ exit(EXIT_FAILURE);
+ }
+ }
+ if (bytes == (size_t) -1) {
+ exit(EXIT_FAILURE);
+ }
+
+ my_close(tmp_file->fd, MYF(MY_WME));
+ ds_close(dst_file);
+
+ list = list_rest(list);
+ my_free(tmp_file->file);
+ }
+
+ pthread_mutex_destroy(&tmpfile_ctxt->mutex);
+
+ my_free(buf);
+ my_free(ctxt->root);
+ my_free(ctxt);
+}
diff --git a/extra/mariabackup/ds_tmpfile.h b/extra/mariabackup/ds_tmpfile.h
new file mode 100644
index 00000000000..c21f1a3f0b5
--- /dev/null
+++ b/extra/mariabackup/ds_tmpfile.h
@@ -0,0 +1,30 @@
+/******************************************************
+Copyright (c) 2012 Percona LLC and/or its affiliates.
+
+tmpfile datasink for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#ifndef DS_TMPFILE_H
+#define DS_TMPFILE_H
+
+#include "datasink.h"
+
+extern datasink_t datasink_tmpfile;
+
+extern MY_TMPDIR mysql_tmpdir_list;
+
+#endif
diff --git a/extra/mariabackup/ds_xbstream.c b/extra/mariabackup/ds_xbstream.c
new file mode 100644
index 00000000000..42924a72d7f
--- /dev/null
+++ b/extra/mariabackup/ds_xbstream.c
@@ -0,0 +1,223 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+Streaming implementation for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#include
+#include
+#include "common.h"
+#include "datasink.h"
+#include "xbstream.h"
+
+typedef struct {
+ xb_wstream_t *xbstream;
+ ds_file_t *dest_file;
+ pthread_mutex_t mutex;
+} ds_stream_ctxt_t;
+
+typedef struct {
+ xb_wstream_file_t *xbstream_file;
+ ds_stream_ctxt_t *stream_ctxt;
+} ds_stream_file_t;
+
+/***********************************************************************
+General streaming interface */
+
+static ds_ctxt_t *xbstream_init(const char *root);
+static ds_file_t *xbstream_open(ds_ctxt_t *ctxt, const char *path,
+ MY_STAT *mystat);
+static int xbstream_write(ds_file_t *file, const void *buf, size_t len);
+static int xbstream_close(ds_file_t *file);
+static void xbstream_deinit(ds_ctxt_t *ctxt);
+
+datasink_t datasink_xbstream = {
+ &xbstream_init,
+ &xbstream_open,
+ &xbstream_write,
+ &xbstream_close,
+ &xbstream_deinit
+};
+
+static
+ssize_t
+my_xbstream_write_callback(xb_wstream_file_t *f __attribute__((unused)),
+ void *userdata, const void *buf, size_t len)
+{
+ ds_stream_ctxt_t *stream_ctxt;
+
+ stream_ctxt = (ds_stream_ctxt_t *) userdata;
+
+ xb_ad(stream_ctxt != NULL);
+ xb_ad(stream_ctxt->dest_file != NULL);
+
+ if (!ds_write(stream_ctxt->dest_file, buf, len)) {
+ return len;
+ }
+ return -1;
+}
+
+static
+ds_ctxt_t *
+xbstream_init(const char *root __attribute__((unused)))
+{
+ ds_ctxt_t *ctxt;
+ ds_stream_ctxt_t *stream_ctxt;
+ xb_wstream_t *xbstream;
+
+ ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_stream_ctxt_t),
+ MYF(MY_FAE));
+ stream_ctxt = (ds_stream_ctxt_t *)(ctxt + 1);
+
+ if (pthread_mutex_init(&stream_ctxt->mutex, NULL)) {
+ msg("xbstream_init: pthread_mutex_init() failed.\n");
+ goto err;
+ }
+
+ xbstream = xb_stream_write_new();
+ if (xbstream == NULL) {
+ msg("xb_stream_write_new() failed.\n");
+ goto err;
+ }
+ stream_ctxt->xbstream = xbstream;
+ stream_ctxt->dest_file = NULL;
+
+ ctxt->ptr = stream_ctxt;
+
+ return ctxt;
+
+err:
+ my_free(ctxt);
+ return NULL;
+}
+
+static
+ds_file_t *
+xbstream_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
+{
+ ds_file_t *file;
+ ds_stream_file_t *stream_file;
+ ds_stream_ctxt_t *stream_ctxt;
+ ds_ctxt_t *dest_ctxt;
+ xb_wstream_t *xbstream;
+ xb_wstream_file_t *xbstream_file;
+
+
+ xb_ad(ctxt->pipe_ctxt != NULL);
+ dest_ctxt = ctxt->pipe_ctxt;
+
+ stream_ctxt = (ds_stream_ctxt_t *) ctxt->ptr;
+
+ pthread_mutex_lock(&stream_ctxt->mutex);
+ if (stream_ctxt->dest_file == NULL) {
+ stream_ctxt->dest_file = ds_open(dest_ctxt, path, mystat);
+ if (stream_ctxt->dest_file == NULL) {
+ return NULL;
+ }
+ }
+ pthread_mutex_unlock(&stream_ctxt->mutex);
+
+ file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
+ sizeof(ds_stream_file_t),
+ MYF(MY_FAE));
+ stream_file = (ds_stream_file_t *) (file + 1);
+
+ xbstream = stream_ctxt->xbstream;
+
+ xbstream_file = xb_stream_write_open(xbstream, path, mystat,
+ stream_ctxt,
+ my_xbstream_write_callback);
+
+ if (xbstream_file == NULL) {
+ msg("xb_stream_write_open() failed.\n");
+ goto err;
+ }
+
+ stream_file->xbstream_file = xbstream_file;
+ stream_file->stream_ctxt = stream_ctxt;
+ file->ptr = stream_file;
+ file->path = stream_ctxt->dest_file->path;
+
+ return file;
+
+err:
+ if (stream_ctxt->dest_file) {
+ ds_close(stream_ctxt->dest_file);
+ stream_ctxt->dest_file = NULL;
+ }
+ my_free(file);
+
+ return NULL;
+}
+
+static
+int
+xbstream_write(ds_file_t *file, const void *buf, size_t len)
+{
+ ds_stream_file_t *stream_file;
+ xb_wstream_file_t *xbstream_file;
+
+
+ stream_file = (ds_stream_file_t *) file->ptr;
+
+ xbstream_file = stream_file->xbstream_file;
+
+ if (xb_stream_write_data(xbstream_file, buf, len)) {
+ msg("xb_stream_write_data() failed.\n");
+ return 1;
+ }
+
+ return 0;
+}
+
+static
+int
+xbstream_close(ds_file_t *file)
+{
+ ds_stream_file_t *stream_file;
+ int rc = 0;
+
+ stream_file = (ds_stream_file_t *)file->ptr;
+
+ rc = xb_stream_write_close(stream_file->xbstream_file);
+
+ my_free(file);
+
+ return rc;
+}
+
+static
+void
+xbstream_deinit(ds_ctxt_t *ctxt)
+{
+ ds_stream_ctxt_t *stream_ctxt;
+
+ stream_ctxt = (ds_stream_ctxt_t *) ctxt->ptr;
+
+ if (xb_stream_write_done(stream_ctxt->xbstream)) {
+ msg("xb_stream_done() failed.\n");
+ }
+
+ if (stream_ctxt->dest_file) {
+ ds_close(stream_ctxt->dest_file);
+ stream_ctxt->dest_file = NULL;
+ }
+
+ pthread_mutex_destroy(&stream_ctxt->mutex);
+
+ my_free(ctxt);
+}
diff --git a/extra/mariabackup/ds_xbstream.h b/extra/mariabackup/ds_xbstream.h
new file mode 100644
index 00000000000..30f34ac8318
--- /dev/null
+++ b/extra/mariabackup/ds_xbstream.h
@@ -0,0 +1,28 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+Streaming interface for XtraBackup.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#ifndef DS_XBSTREAM_H
+#define DS_XBSTREAM_H
+
+#include "datasink.h"
+
+extern datasink_t datasink_xbstream;
+
+#endif
diff --git a/extra/mariabackup/encryption_plugin.cc b/extra/mariabackup/encryption_plugin.cc
new file mode 100644
index 00000000000..76512c185e2
--- /dev/null
+++ b/extra/mariabackup/encryption_plugin.cc
@@ -0,0 +1,157 @@
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+
+extern struct st_maria_plugin *mysql_optional_plugins[];
+extern struct st_maria_plugin *mysql_mandatory_plugins[];
+static void encryption_plugin_init(int argc, char **argv);
+
+extern char *xb_plugin_load;
+extern char *xb_plugin_dir;
+
+const int PLUGIN_MAX_ARGS = 1024;
+std::vector backup_plugins_args;
+
+const char *QUERY_PLUGIN =
+"SELECT plugin_name, plugin_library, @@plugin_dir"
+" FROM information_schema.plugins WHERE plugin_type='ENCRYPTION'"
+" AND plugin_status='ACTIVE'";
+
+std::string encryption_plugin_config;
+
+static void add_to_plugin_load_list(const char *plugin_def)
+{
+ opt_plugin_load_list_ptr->push_back(new i_string(plugin_def));
+}
+
+static char XTRABACKUP_EXE[] = "xtrabackup";
+
+void encryption_plugin_backup_init(MYSQL *mysql)
+{
+ MYSQL_RES *result;
+ MYSQL_ROW row;
+ std::ostringstream oss;
+ char *argv[PLUGIN_MAX_ARGS];
+ int argc;
+
+ result = xb_mysql_query(mysql, QUERY_PLUGIN, true, true);
+ row = mysql_fetch_row(result);
+ if (!row)
+ {
+ mysql_free_result(result);
+ return;
+ }
+
+ char *name= row[0];
+ char *library= row[1];
+ char *dir= row[2];
+
+#ifdef _WIN32
+ for (char *p = dir; *p; p++)
+ if (*p == '\\') *p = '/';
+#endif
+
+ std::string plugin_load(name);
+ if (library)
+ plugin_load += std::string("=") + library;
+
+ oss << "plugin_load=" << plugin_load << std::endl;
+
+ /* Required to load the plugin later.*/
+ add_to_plugin_load_list(plugin_load.c_str());
+ strncpy(opt_plugin_dir, dir, FN_REFLEN);
+
+ oss << "plugin_dir=" << '"' << dir << '"' << std::endl;
+
+
+ /* Read plugin variables. */
+ char query[1024];
+ snprintf(query, 1024, "SHOW variables like '%s_%%'", name);
+ mysql_free_result(result);
+
+ result = xb_mysql_query(mysql, query, true, true);
+ while ((row = mysql_fetch_row(result)))
+ {
+ std::string arg("--");
+ arg += row[0];
+ arg += "=";
+ arg += row[1];
+ backup_plugins_args.push_back(arg);
+ oss << row[0] << "=" << row[1] << std::endl;
+ }
+
+ mysql_free_result(result);
+
+ /* Check whether to encrypt logs. */
+ result = xb_mysql_query(mysql, "select @@innodb_encrypt_log", true, true);
+ row = mysql_fetch_row(result);
+ srv_encrypt_log = (row != 0 && row[0][0] == '1');
+ oss << "innodb_encrypt_log=" << row[0] << std::endl;
+
+ mysql_free_result(result);
+
+ encryption_plugin_config = oss.str();
+
+ argc = 0;
+ argv[argc++] = XTRABACKUP_EXE;
+ for(size_t i = 0; i < backup_plugins_args.size(); i++)
+ {
+ argv[argc++] = (char *)backup_plugins_args[i].c_str();
+ if (argc == PLUGIN_MAX_ARGS - 2)
+ break;
+ }
+ argv[argc] = 0;
+
+ encryption_plugin_init(argc, argv);
+}
+
+const char *encryption_plugin_get_config()
+{
+ return encryption_plugin_config.c_str();
+}
+
+extern int finalize_encryption_plugin(st_plugin_int *plugin);
+
+
+void encryption_plugin_prepare_init(int argc, char **argv)
+{
+
+ if (!xb_plugin_load)
+ {
+ finalize_encryption_plugin(0);
+ return;
+ }
+
+ add_to_plugin_load_list(xb_plugin_load);
+
+ if (xb_plugin_dir)
+ strncpy(opt_plugin_dir, xb_plugin_dir, FN_REFLEN);
+
+ char **new_argv = new char *[argc + 1];
+ new_argv[0] = XTRABACKUP_EXE;
+ memcpy(&new_argv[1], argv, argc*sizeof(char *));
+
+ encryption_plugin_init(argc+1, new_argv);
+
+ delete[] new_argv;
+}
+
+static void encryption_plugin_init(int argc, char **argv)
+{
+ /* Patch optional and mandatory plugins, we only need to load the one in xb_plugin_load. */
+ mysql_optional_plugins[0] = mysql_mandatory_plugins[0] = 0;
+ msg("Loading encryption plugin\n");
+ for (int i= 1; i < argc; i++)
+ msg("\t Encryption plugin parameter : '%s'\n", argv[i]);
+ plugin_init(&argc, argv, PLUGIN_INIT_SKIP_PLUGIN_TABLE);
+}
+
diff --git a/extra/mariabackup/encryption_plugin.h b/extra/mariabackup/encryption_plugin.h
new file mode 100644
index 00000000000..16d74790254
--- /dev/null
+++ b/extra/mariabackup/encryption_plugin.h
@@ -0,0 +1,7 @@
+#include
+#include
+extern void encryption_plugin_backup_init(MYSQL *mysql);
+extern const char* encryption_plugin_get_config();
+extern void encryption_plugin_prepare_init(int argc, char **argv);
+
+//extern void encryption_plugin_init(int argc, char **argv);
diff --git a/extra/mariabackup/fil_cur.cc b/extra/mariabackup/fil_cur.cc
new file mode 100644
index 00000000000..03f4ce0d7a7
--- /dev/null
+++ b/extra/mariabackup/fil_cur.cc
@@ -0,0 +1,402 @@
+/******************************************************
+XtraBackup: hot backup tool for InnoDB
+(c) 2009-2013 Percona LLC and/or its affiliates.
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+/* Source file cursor implementation */
+
+#include
+
+#include
+#include
+#include
+#include
+
+#include "fil_cur.h"
+#include "common.h"
+#include "read_filt.h"
+#include "xtrabackup.h"
+#include "xb0xb.h"
+
+/* Size of read buffer in pages (640 pages = 10M for 16K sized pages) */
+#define XB_FIL_CUR_PAGES 640
+
+/***********************************************************************
+Extracts the relative path ("database/table.ibd") of a tablespace from a
+specified possibly absolute path.
+
+For user tablespaces both "./database/table.ibd" and
+"/remote/dir/database/table.ibd" result in "database/table.ibd".
+
+For system tablepsaces (i.e. When is_system is TRUE) both "/remote/dir/ibdata1"
+and "./ibdata1" yield "ibdata1" in the output. */
+const char *
+xb_get_relative_path(
+/*=================*/
+ const char* path, /*!< in: tablespace path (either
+ relative or absolute) */
+ ibool is_system) /*!< in: TRUE for system tablespaces,
+ i.e. when only the filename must be
+ returned. */
+{
+ const char *next;
+ const char *cur;
+ const char *prev;
+
+ prev = NULL;
+ cur = path;
+
+ while ((next = strchr(cur, OS_PATH_SEPARATOR)) != NULL) {
+
+ prev = cur;
+ cur = next + 1;
+ }
+
+ if (is_system) {
+
+ return(cur);
+ } else {
+
+ return((prev == NULL) ? cur : prev);
+ }
+
+}
+
+/**********************************************************************//**
+Closes a file. */
+static
+void
+xb_fil_node_close_file(
+/*===================*/
+ fil_node_t* node) /*!< in: file node */
+{
+ ibool ret;
+
+ mutex_enter(&fil_system->mutex);
+
+ ut_ad(node);
+ ut_a(node->n_pending == 0);
+ ut_a(node->n_pending_flushes == 0);
+ ut_a(!node->being_extended);
+
+ if (!node->is_open()) {
+
+ mutex_exit(&fil_system->mutex);
+
+ return;
+ }
+
+ ret = os_file_close(node->handle);
+ ut_a(ret);
+
+ node->handle = OS_FILE_CLOSED;
+
+ ut_a(fil_system->n_open > 0);
+ fil_system->n_open--;
+ fil_n_file_opened--;
+
+ if (node->space->purpose == FIL_TYPE_TABLESPACE &&
+ fil_is_user_tablespace_id(node->space->id)) {
+
+ ut_a(UT_LIST_GET_LEN(fil_system->LRU) > 0);
+
+ /* The node is in the LRU list, remove it */
+ UT_LIST_REMOVE(fil_system->LRU, node);
+ }
+
+ mutex_exit(&fil_system->mutex);
+}
+
+/************************************************************************
+Open a source file cursor and initialize the associated read filter.
+
+@return XB_FIL_CUR_SUCCESS on success, XB_FIL_CUR_SKIP if the source file must
+be skipped and XB_FIL_CUR_ERROR on error. */
+xb_fil_cur_result_t
+xb_fil_cur_open(
+/*============*/
+ xb_fil_cur_t* cursor, /*!< out: source file cursor */
+ xb_read_filt_t* read_filter, /*!< in/out: the read filter */
+ fil_node_t* node, /*!< in: source tablespace node */
+ uint thread_n) /*!< thread number for diagnostics */
+{
+ bool success;
+
+ /* Initialize these first so xb_fil_cur_close() handles them correctly
+ in case of error */
+ cursor->orig_buf = NULL;
+ cursor->node = NULL;
+
+ cursor->space_id = node->space->id;
+
+ strncpy(cursor->abs_path, node->name, sizeof(cursor->abs_path));
+
+ /* Get the relative path for the destination tablespace name, i.e. the
+ one that can be appended to the backup root directory. Non-system
+ tablespaces may have absolute paths for DATA DIRECTORY.
+ We want to make "local" copies for the backup. */
+ strncpy(cursor->rel_path,
+ xb_get_relative_path(cursor->abs_path, cursor->is_system()),
+ sizeof(cursor->rel_path));
+
+ /* In the backup mode we should already have a tablespace handle created
+ by fil_ibd_load() unless it is a system
+ tablespace. Otherwise we open the file here. */
+ if (cursor->is_system() || srv_operation == SRV_OPERATION_RESTORE_DELTA
+ || xb_close_files) {
+ node->handle = os_file_create_simple_no_error_handling(
+ 0, node->name,
+ OS_FILE_OPEN,
+ OS_FILE_READ_ALLOW_DELETE, true, &success);
+ if (!success) {
+ /* The following call prints an error message */
+ os_file_get_last_error(TRUE);
+
+ msg("[%02u] xtrabackup: error: cannot open "
+ "tablespace %s\n",
+ thread_n, cursor->abs_path);
+
+ return(XB_FIL_CUR_ERROR);
+ }
+ mutex_enter(&fil_system->mutex);
+
+ fil_system->n_open++;
+ fil_n_file_opened++;
+
+ if (node->space->purpose == FIL_TYPE_TABLESPACE &&
+ fil_is_user_tablespace_id(node->space->id)) {
+
+ /* Put the node to the LRU list */
+ UT_LIST_ADD_FIRST(fil_system->LRU, node);
+ }
+
+ mutex_exit(&fil_system->mutex);
+ }
+
+ ut_ad(node->is_open());
+
+ cursor->node = node;
+ cursor->file = node->handle;
+
+ if (stat(cursor->abs_path, &cursor->statinfo)) {
+ msg("[%02u] xtrabackup: error: cannot stat %s\n",
+ thread_n, cursor->abs_path);
+
+ xb_fil_cur_close(cursor);
+
+ return(XB_FIL_CUR_ERROR);
+ }
+
+ if (srv_file_flush_method == SRV_O_DIRECT
+ || srv_file_flush_method == SRV_O_DIRECT_NO_FSYNC) {
+
+ os_file_set_nocache(cursor->file, node->name, "OPEN");
+ }
+
+ posix_fadvise(cursor->file, 0, 0, POSIX_FADV_SEQUENTIAL);
+
+ /* Determine the page size */
+ ulint flags = xb_get_space_flags(cursor->file);
+ if (flags == ULINT_UNDEFINED) {
+ xb_fil_cur_close(cursor);
+ return(XB_FIL_CUR_SKIP);
+ }
+
+ if (!fsp_flags_is_valid(flags, cursor->space_id)) {
+ ulint cflags = fsp_flags_convert_from_101(flags);
+ if (cflags == ULINT_UNDEFINED) {
+ msg("[%02u] xtrabackup: Error: Invalid "
+ "tablespace flags: %x.\n", thread_n, uint(flags));
+ return(XB_FIL_CUR_SKIP);
+ }
+ flags = cflags;
+ }
+
+ const page_size_t page_size(flags);
+ cursor->page_size = page_size;
+
+ /* Allocate read buffer */
+ cursor->buf_size = XB_FIL_CUR_PAGES * page_size.physical();
+ cursor->orig_buf = static_cast
+ (malloc(cursor->buf_size + UNIV_PAGE_SIZE));
+ cursor->buf = static_cast
+ (ut_align(cursor->orig_buf, UNIV_PAGE_SIZE));
+
+ cursor->buf_read = 0;
+ cursor->buf_npages = 0;
+ cursor->buf_offset = 0;
+ cursor->buf_page_no = 0;
+ cursor->thread_n = thread_n;
+
+ cursor->space_size = (ulint)(cursor->statinfo.st_size
+ / page_size.physical());
+
+ cursor->read_filter = read_filter;
+ cursor->read_filter->init(&cursor->read_filter_ctxt, cursor,
+ node->space->id);
+
+ return(XB_FIL_CUR_SUCCESS);
+}
+
+/************************************************************************
+Reads and verifies the next block of pages from the source
+file. Positions the cursor after the last read non-corrupted page.
+
+@return XB_FIL_CUR_SUCCESS if some have been read successfully, XB_FIL_CUR_EOF
+if there are no more pages to read and XB_FIL_CUR_ERROR on error. */
+xb_fil_cur_result_t
+xb_fil_cur_read(
+/*============*/
+ xb_fil_cur_t* cursor) /*!< in/out: source file cursor */
+{
+ ibool success;
+ byte* page;
+ ulint i;
+ ulint npages;
+ ulint retry_count;
+ xb_fil_cur_result_t ret;
+ ib_int64_t offset;
+ ib_int64_t to_read;
+ const ulint page_size = cursor->page_size.physical();
+ xb_ad(!cursor->is_system() || page_size == UNIV_PAGE_SIZE);
+
+ cursor->read_filter->get_next_batch(&cursor->read_filter_ctxt,
+ &offset, &to_read);
+
+ if (to_read == 0LL) {
+ return(XB_FIL_CUR_EOF);
+ }
+
+ if (to_read > (ib_int64_t) cursor->buf_size) {
+ to_read = (ib_int64_t) cursor->buf_size;
+ }
+
+ xb_a(to_read > 0 && to_read <= 0xFFFFFFFFLL);
+
+ if ((to_read & ~(page_size - 1))
+ && offset + to_read == cursor->statinfo.st_size) {
+
+ if (to_read < (ib_int64_t) page_size) {
+ msg("[%02u] xtrabackup: Warning: junk at the end of "
+ "%s:\n", cursor->thread_n, cursor->abs_path);
+ msg("[%02u] xtrabackup: Warning: offset = %llu, "
+ "to_read = %llu\n",
+ cursor->thread_n,
+ (unsigned long long) offset,
+ (unsigned long long) to_read);
+
+ return(XB_FIL_CUR_EOF);
+ }
+
+ to_read = (ib_int64_t) (((ulint) to_read) &
+ ~(page_size - 1));
+ }
+
+ xb_a((to_read & (page_size - 1)) == 0);
+
+ npages = (ulint) (to_read / cursor->page_size.physical());
+
+ retry_count = 10;
+ ret = XB_FIL_CUR_SUCCESS;
+
+read_retry:
+ xtrabackup_io_throttling();
+
+ cursor->buf_read = 0;
+ cursor->buf_npages = 0;
+ cursor->buf_offset = offset;
+ cursor->buf_page_no = (ulint)(offset / cursor->page_size.physical());
+
+ FilSpace space(cursor->space_id);
+
+ if (!space()) {
+ return(XB_FIL_CUR_ERROR);
+ }
+
+ success = os_file_read(IORequestRead,
+ cursor->file, cursor->buf, offset,
+ (ulint) to_read);
+ if (!success) {
+ return(XB_FIL_CUR_ERROR);
+ }
+
+ /* check pages for corruption and re-read if necessary. i.e. in case of
+ partially written pages */
+ for (page = cursor->buf, i = 0; i < npages;
+ page += page_size, i++) {
+ ulint page_no = cursor->buf_page_no + i;
+
+ if (cursor->space_id == TRX_SYS_SPACE &&
+ page_no >= FSP_EXTENT_SIZE &&
+ page_no < FSP_EXTENT_SIZE * 3) {
+ /* We ignore the doublewrite buffer pages */
+ } else if (!fil_space_verify_crypt_checksum(
+ page, cursor->page_size, space->id, page_no)
+ && buf_page_is_corrupted(true, page,
+ cursor->page_size,
+ space)) {
+ retry_count--;
+ if (retry_count == 0) {
+ msg("[%02u] xtrabackup: "
+ "Error: failed to read page after "
+ "10 retries. File %s seems to be "
+ "corrupted.\n", cursor->thread_n,
+ cursor->abs_path);
+ ret = XB_FIL_CUR_ERROR;
+ break;
+ }
+
+ if (retry_count == 9) {
+ msg("[%02u] xtrabackup: "
+ "Database page corruption detected at page "
+ ULINTPF ", retrying...\n",
+ cursor->thread_n, page_no);
+ }
+
+ os_thread_sleep(100000);
+
+ goto read_retry;
+ }
+ cursor->buf_read += page_size;
+ cursor->buf_npages++;
+ }
+
+ posix_fadvise(cursor->file, offset, to_read, POSIX_FADV_DONTNEED);
+
+ return(ret);
+}
+
+/************************************************************************
+Close the source file cursor opened with xb_fil_cur_open() and its
+associated read filter. */
+void
+xb_fil_cur_close(
+/*=============*/
+ xb_fil_cur_t *cursor) /*!< in/out: source file cursor */
+{
+ cursor->read_filter->deinit(&cursor->read_filter_ctxt);
+
+ free(cursor->orig_buf);
+
+ if (cursor->node != NULL) {
+ xb_fil_node_close_file(cursor->node);
+ cursor->file = OS_FILE_CLOSED;
+ }
+}
diff --git a/extra/mariabackup/fil_cur.h b/extra/mariabackup/fil_cur.h
new file mode 100644
index 00000000000..e3f356a346c
--- /dev/null
+++ b/extra/mariabackup/fil_cur.h
@@ -0,0 +1,128 @@
+/******************************************************
+XtraBackup: hot backup tool for InnoDB
+(c) 2009-2013 Percona LLC and/or its affiliates.
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+/* Source file cursor interface */
+
+#ifndef FIL_CUR_H
+#define FIL_CUR_H
+
+#include
+#include "read_filt.h"
+#include "srv0start.h"
+
+struct xb_fil_cur_t {
+ pfs_os_file_t file; /*!< source file handle */
+ fil_node_t* node; /*!< source tablespace node */
+ char rel_path[FN_REFLEN];
+ /*!< normalized file path */
+ char abs_path[FN_REFLEN];
+ /*!< absolute file path */
+ MY_STAT statinfo; /*!< information about the file */
+ page_size_t page_size; /*!< page size */
+ xb_read_filt_t* read_filter; /*!< read filter */
+ xb_read_filt_ctxt_t read_filter_ctxt;
+ /*!< read filter context */
+ byte* orig_buf; /*!< read buffer */
+ byte* buf; /*!< aligned pointer for orig_buf */
+ size_t buf_size; /*!< buffer size in bytes */
+ size_t buf_read; /*!< number of read bytes in buffer
+ after the last cursor read */
+ size_t buf_npages; /*!< number of pages in buffer after the
+ last cursor read */
+ ib_int64_t buf_offset; /*!< file offset of the first page in
+ buffer */
+ ulint buf_page_no; /*!< number of the first page in
+ buffer */
+ uint thread_n; /*!< thread number for diagnostics */
+ ulint space_id; /*!< ID of tablespace */
+ ulint space_size; /*!< space size in pages */
+
+ /** TODO: remove this default constructor */
+ xb_fil_cur_t() : page_size(0), read_filter_ctxt() {}
+
+ /** @return whether this is not a file-per-table tablespace */
+ bool is_system() const
+ {
+ ut_ad(space_id != SRV_TMP_SPACE_ID);
+ return(space_id == TRX_SYS_SPACE
+ || srv_is_undo_tablespace(space_id));
+ }
+};
+
+typedef enum {
+ XB_FIL_CUR_SUCCESS,
+ XB_FIL_CUR_SKIP,
+ XB_FIL_CUR_ERROR,
+ XB_FIL_CUR_EOF
+} xb_fil_cur_result_t;
+
+/************************************************************************
+Open a source file cursor and initialize the associated read filter.
+
+@return XB_FIL_CUR_SUCCESS on success, XB_FIL_CUR_SKIP if the source file must
+be skipped and XB_FIL_CUR_ERROR on error. */
+xb_fil_cur_result_t
+xb_fil_cur_open(
+/*============*/
+ xb_fil_cur_t* cursor, /*!< out: source file cursor */
+ xb_read_filt_t* read_filter, /*!< in/out: the read filter */
+ fil_node_t* node, /*!< in: source tablespace node */
+ uint thread_n); /*!< thread number for diagnostics */
+
+/************************************************************************
+Reads and verifies the next block of pages from the source
+file. Positions the cursor after the last read non-corrupted page.
+
+@return XB_FIL_CUR_SUCCESS if some have been read successfully, XB_FIL_CUR_EOF
+if there are no more pages to read and XB_FIL_CUR_ERROR on error. */
+xb_fil_cur_result_t
+xb_fil_cur_read(
+/*============*/
+ xb_fil_cur_t* cursor); /*!< in/out: source file cursor */
+
+/************************************************************************
+Close the source file cursor opened with xb_fil_cur_open() and its
+associated read filter. */
+void
+xb_fil_cur_close(
+/*=============*/
+ xb_fil_cur_t *cursor); /*!< in/out: source file cursor */
+
+/***********************************************************************
+Extracts the relative path ("database/table.ibd") of a tablespace from a
+specified possibly absolute path.
+
+For user tablespaces both "./database/table.ibd" and
+"/remote/dir/database/table.ibd" result in "database/table.ibd".
+
+For system tablepsaces (i.e. When is_system is TRUE) both "/remote/dir/ibdata1"
+and "./ibdata1" yield "ibdata1" in the output. */
+const char *
+xb_get_relative_path(
+/*=================*/
+ const char* path, /*!< in: tablespace path (either
+ relative or absolute) */
+ ibool is_system); /*!< in: TRUE for system tablespaces,
+ i.e. when only the filename must be
+ returned. */
+
+#endif
diff --git a/extra/mariabackup/innobackupex.cc b/extra/mariabackup/innobackupex.cc
new file mode 100644
index 00000000000..c15e02cdff3
--- /dev/null
+++ b/extra/mariabackup/innobackupex.cc
@@ -0,0 +1,1012 @@
+/******************************************************
+hot backup tool for InnoDB
+(c) 2009-2015 Percona LLC and/or its affiliates
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************
+
+This file incorporates work covered by the following copyright and
+permission notice:
+
+Copyright (c) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place, Suite 330, Boston, MA 02111-1307 USA
+
+*******************************************************/
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include "common.h"
+#include "innobackupex.h"
+#include "xtrabackup.h"
+#include "xbstream.h"
+#include "fil_cur.h"
+#include "write_filt.h"
+#include "backup_copy.h"
+
+using std::min;
+using std::max;
+
+/* options */
+my_bool opt_ibx_version = FALSE;
+my_bool opt_ibx_help = FALSE;
+my_bool opt_ibx_apply_log = FALSE;
+my_bool opt_ibx_incremental = FALSE;
+my_bool opt_ibx_notimestamp = FALSE;
+
+my_bool opt_ibx_copy_back = FALSE;
+my_bool opt_ibx_move_back = FALSE;
+my_bool opt_ibx_galera_info = FALSE;
+my_bool opt_ibx_slave_info = FALSE;
+my_bool opt_ibx_no_lock = FALSE;
+my_bool opt_ibx_safe_slave_backup = FALSE;
+my_bool opt_ibx_rsync = FALSE;
+my_bool opt_ibx_force_non_empty_dirs = FALSE;
+my_bool opt_ibx_noversioncheck = FALSE;
+my_bool opt_ibx_no_backup_locks = FALSE;
+my_bool opt_ibx_decompress = FALSE;
+
+char *opt_ibx_incremental_history_name = NULL;
+char *opt_ibx_incremental_history_uuid = NULL;
+
+char *opt_ibx_user = NULL;
+char *opt_ibx_password = NULL;
+char *opt_ibx_host = NULL;
+char *opt_ibx_defaults_group = NULL;
+char *opt_ibx_socket = NULL;
+uint opt_ibx_port = 0;
+
+ulong opt_ibx_lock_wait_query_type;
+ulong opt_ibx_kill_long_query_type;
+
+uint opt_ibx_kill_long_queries_timeout = 0;
+uint opt_ibx_lock_wait_timeout = 0;
+uint opt_ibx_lock_wait_threshold = 0;
+uint opt_ibx_debug_sleep_before_unlock = 0;
+uint opt_ibx_safe_slave_backup_timeout = 0;
+
+const char *opt_ibx_history = NULL;
+
+char *opt_ibx_include = NULL;
+char *opt_ibx_databases = NULL;
+bool ibx_partial_backup = false;
+
+char *ibx_position_arg = NULL;
+char *ibx_backup_directory = NULL;
+
+/* copy of proxied xtrabackup options */
+my_bool ibx_xb_close_files;
+const char *ibx_xtrabackup_compress_alg;
+uint ibx_xtrabackup_compress_threads;
+ulonglong ibx_xtrabackup_compress_chunk_size;
+my_bool ibx_xtrabackup_export;
+char *ibx_xtrabackup_extra_lsndir;
+char *ibx_xtrabackup_incremental_basedir;
+char *ibx_xtrabackup_incremental_dir;
+my_bool ibx_xtrabackup_incremental_force_scan;
+ulint ibx_xtrabackup_log_copy_interval;
+char *ibx_xtrabackup_incremental;
+int ibx_xtrabackup_parallel;
+char *ibx_xtrabackup_stream_str;
+char *ibx_xtrabackup_tables_file;
+long ibx_xtrabackup_throttle;
+char *ibx_opt_mysql_tmpdir;
+longlong ibx_xtrabackup_use_memory;
+
+
+static inline int ibx_msg(const char *fmt, ...) ATTRIBUTE_FORMAT(printf, 1, 2);
+static inline int ibx_msg(const char *fmt, ...)
+{
+ int result;
+ time_t t = time(NULL);
+ char date[100];
+ char *line;
+ va_list args;
+
+ strftime(date, sizeof(date), "%y%m%d %H:%M:%S", localtime(&t));
+
+ va_start(args, fmt);
+
+ result = vasprintf(&line, fmt, args);
+
+ va_end(args);
+
+ if (result != -1) {
+ result = fprintf(stderr, "%s %s: %s",
+ date, INNOBACKUPEX_BIN_NAME, line);
+ free(line);
+ }
+
+ return result;
+}
+
+enum innobackupex_options
+{
+ OPT_APPLY_LOG = 256,
+ OPT_COPY_BACK,
+ OPT_MOVE_BACK,
+ OPT_REDO_ONLY,
+ OPT_GALERA_INFO,
+ OPT_SLAVE_INFO,
+ OPT_INCREMENTAL,
+ OPT_INCREMENTAL_HISTORY_NAME,
+ OPT_INCREMENTAL_HISTORY_UUID,
+ OPT_LOCK_WAIT_QUERY_TYPE,
+ OPT_KILL_LONG_QUERY_TYPE,
+ OPT_KILL_LONG_QUERIES_TIMEOUT,
+ OPT_LOCK_WAIT_TIMEOUT,
+ OPT_LOCK_WAIT_THRESHOLD,
+ OPT_DEBUG_SLEEP_BEFORE_UNLOCK,
+ OPT_NO_LOCK,
+ OPT_SAFE_SLAVE_BACKUP,
+ OPT_SAFE_SLAVE_BACKUP_TIMEOUT,
+ OPT_RSYNC,
+ OPT_HISTORY,
+ OPT_INCLUDE,
+ OPT_FORCE_NON_EMPTY_DIRS,
+ OPT_NO_TIMESTAMP,
+ OPT_NO_VERSION_CHECK,
+ OPT_NO_BACKUP_LOCKS,
+ OPT_DATABASES,
+ OPT_DECOMPRESS,
+
+ /* options wich are passed directly to xtrabackup */
+ OPT_CLOSE_FILES,
+ OPT_COMPACT,
+ OPT_COMPRESS,
+ OPT_COMPRESS_THREADS,
+ OPT_COMPRESS_CHUNK_SIZE,
+ OPT_EXPORT,
+ OPT_EXTRA_LSNDIR,
+ OPT_INCREMENTAL_BASEDIR,
+ OPT_INCREMENTAL_DIR,
+ OPT_INCREMENTAL_FORCE_SCAN,
+ OPT_LOG_COPY_INTERVAL,
+ OPT_PARALLEL,
+ OPT_REBUILD_INDEXES,
+ OPT_REBUILD_THREADS,
+ OPT_STREAM,
+ OPT_TABLES_FILE,
+ OPT_THROTTLE,
+ OPT_USE_MEMORY
+};
+
+ibx_mode_t ibx_mode = IBX_MODE_BACKUP;
+
+static struct my_option ibx_long_options[] =
+{
+ {"version", 'v', "print xtrabackup version information",
+ (uchar *) &opt_ibx_version, (uchar *) &opt_ibx_version, 0,
+ GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"help", '?', "This option displays a help screen and exits.",
+ (uchar *) &opt_ibx_help, (uchar *) &opt_ibx_help, 0,
+ GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"apply-log", OPT_APPLY_LOG, "Prepare a backup in BACKUP-DIR by "
+ "applying the redo log 'ib_logfile0' and creating new redo log. "
+ "The InnoDB configuration is read from the file \"backup-my.cnf\".",
+ (uchar*) &opt_ibx_apply_log, (uchar*) &opt_ibx_apply_log,
+ 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"copy-back", OPT_COPY_BACK, "Copy all the files in a previously made "
+ "backup from the backup directory to their original locations.",
+ (uchar *) &opt_ibx_copy_back, (uchar *) &opt_ibx_copy_back, 0,
+ GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"move-back", OPT_MOVE_BACK, "Move all the files in a previously made "
+ "backup from the backup directory to the actual datadir location. "
+ "Use with caution, as it removes backup files.",
+ (uchar *) &opt_ibx_move_back, (uchar *) &opt_ibx_move_back, 0,
+ GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"galera-info", OPT_GALERA_INFO, "This options creates the "
+ "xtrabackup_galera_info file which contains the local node state at "
+ "the time of the backup. Option should be used when performing the "
+ "backup of Percona-XtraDB-Cluster. Has no effect when backup locks "
+ "are used to create the backup.",
+ (uchar *) &opt_ibx_galera_info, (uchar *) &opt_ibx_galera_info, 0,
+ GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"slave-info", OPT_SLAVE_INFO, "This option is useful when backing "
+ "up a replication slave server. It prints the binary log position "
+ "and name of the master server. It also writes this information to "
+ "the \"xtrabackup_slave_info\" file as a \"CHANGE MASTER\" command. "
+ "A new slave for this master can be set up by starting a slave server "
+ "on this backup and issuing a \"CHANGE MASTER\" command with the "
+ "binary log position saved in the \"xtrabackup_slave_info\" file.",
+ (uchar *) &opt_ibx_slave_info, (uchar *) &opt_ibx_slave_info, 0,
+ GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"incremental", OPT_INCREMENTAL, "This option tells xtrabackup to "
+ "create an incremental backup, rather than a full one. It is passed "
+ "to the xtrabackup child process. When this option is specified, "
+ "either --incremental-lsn or --incremental-basedir can also be given. "
+ "If neither option is given, option --incremental-basedir is passed "
+ "to xtrabackup by default, set to the first timestamped backup "
+ "directory in the backup base directory.",
+ (uchar *) &opt_ibx_incremental, (uchar *) &opt_ibx_incremental, 0,
+ GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"no-lock", OPT_NO_LOCK, "Use this option to disable table lock "
+ "with \"FLUSH TABLES WITH READ LOCK\". Use it only if ALL your "
+ "tables are InnoDB and you DO NOT CARE about the binary log "
+ "position of the backup. This option shouldn't be used if there "
+ "are any DDL statements being executed or if any updates are "
+ "happening on non-InnoDB tables (this includes the system MyISAM "
+ "tables in the mysql database), otherwise it could lead to an "
+ "inconsistent backup. If you are considering to use --no-lock "
+ "because your backups are failing to acquire the lock, this could "
+ "be because of incoming replication events preventing the lock "
+ "from succeeding. Please try using --safe-slave-backup to "
+ "momentarily stop the replication slave thread, this may help "
+ "the backup to succeed and you then don't need to resort to "
+ "using this option.",
+ (uchar *) &opt_ibx_no_lock, (uchar *) &opt_ibx_no_lock, 0,
+ GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"safe-slave-backup", OPT_SAFE_SLAVE_BACKUP, "Stop slave SQL thread "
+ "and wait to start backup until Slave_open_temp_tables in "
+ "\"SHOW STATUS\" is zero. If there are no open temporary tables, "
+ "the backup will take place, otherwise the SQL thread will be "
+ "started and stopped until there are no open temporary tables. "
+ "The backup will fail if Slave_open_temp_tables does not become "
+ "zero after --safe-slave-backup-timeout seconds. The slave SQL "
+ "thread will be restarted when the backup finishes.",
+ (uchar *) &opt_ibx_safe_slave_backup,
+ (uchar *) &opt_ibx_safe_slave_backup,
+ 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"rsync", OPT_RSYNC, "Uses the rsync utility to optimize local file "
+ "transfers. When this option is specified, innobackupex uses rsync "
+ "to copy all non-InnoDB files instead of spawning a separate cp for "
+ "each file, which can be much faster for servers with a large number "
+ "of databases or tables. This option cannot be used together with "
+ "--stream.",
+ (uchar *) &opt_ibx_rsync, (uchar *) &opt_ibx_rsync,
+ 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"force-non-empty-directories", OPT_FORCE_NON_EMPTY_DIRS, "This "
+ "option, when specified, makes --copy-back or --move-back transfer "
+ "files to non-empty directories. Note that no existing files will be "
+ "overwritten. If --copy-back or --nove-back has to copy a file from "
+ "the backup directory which already exists in the destination "
+ "directory, it will still fail with an error.",
+ (uchar *) &opt_ibx_force_non_empty_dirs,
+ (uchar *) &opt_ibx_force_non_empty_dirs,
+ 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"no-timestamp", OPT_NO_TIMESTAMP, "This option prevents creation of a "
+ "time-stamped subdirectory of the BACKUP-ROOT-DIR given on the "
+ "command line. When it is specified, the backup is done in "
+ "BACKUP-ROOT-DIR instead.",
+ (uchar *) &opt_ibx_notimestamp,
+ (uchar *) &opt_ibx_notimestamp,
+ 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"no-version-check", OPT_NO_VERSION_CHECK, "This option disables the "
+ "version check which is enabled by the --version-check option.",
+ (uchar *) &opt_ibx_noversioncheck,
+ (uchar *) &opt_ibx_noversioncheck,
+ 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"no-backup-locks", OPT_NO_BACKUP_LOCKS, "This option controls if "
+ "backup locks should be used instead of FLUSH TABLES WITH READ LOCK "
+ "on the backup stage. The option has no effect when backup locks are "
+ "not supported by the server. This option is enabled by default, "
+ "disable with --no-backup-locks.",
+ (uchar *) &opt_ibx_no_backup_locks,
+ (uchar *) &opt_ibx_no_backup_locks,
+ 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"decompress", OPT_DECOMPRESS, "Decompresses all files with the .qp "
+ "extension in a backup previously made with the --compress option.",
+ (uchar *) &opt_ibx_decompress,
+ (uchar *) &opt_ibx_decompress,
+ 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"user", 'u', "This option specifies the MySQL username used "
+ "when connecting to the server, if that's not the current user. "
+ "The option accepts a string argument. See mysql --help for details.",
+ (uchar*) &opt_ibx_user, (uchar*) &opt_ibx_user, 0, GET_STR,
+ REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"host", 'H', "This option specifies the host to use when "
+ "connecting to the database server with TCP/IP. The option accepts "
+ "a string argument. See mysql --help for details.",
+ (uchar*) &opt_ibx_host, (uchar*) &opt_ibx_host, 0, GET_STR,
+ REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"port", 'P', "This option specifies the port to use when "
+ "connecting to the database server with TCP/IP. The option accepts "
+ "a string argument. See mysql --help for details.",
+ &opt_ibx_port, &opt_ibx_port, 0, GET_UINT, REQUIRED_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"password", 'p', "This option specifies the password to use "
+ "when connecting to the database. It accepts a string argument. "
+ "See mysql --help for details.",
+ 0, 0, 0, GET_STR,
+ REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"socket", 'S', "This option specifies the socket to use when "
+ "connecting to the local database server with a UNIX domain socket. "
+ "The option accepts a string argument. See mysql --help for details.",
+ (uchar*) &opt_ibx_socket, (uchar*) &opt_ibx_socket, 0, GET_STR,
+ REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"incremental-history-name", OPT_INCREMENTAL_HISTORY_NAME,
+ "This option specifies the name of the backup series stored in the "
+ "PERCONA_SCHEMA.xtrabackup_history history record to base an "
+ "incremental backup on. Xtrabackup will search the history table "
+ "looking for the most recent (highest innodb_to_lsn), successful "
+ "backup in the series and take the to_lsn value to use as the "
+ "starting lsn for the incremental backup. This will be mutually "
+ "exclusive with --incremental-history-uuid, --incremental-basedir "
+ "and --incremental-lsn. If no valid lsn can be found (no series by "
+ "that name, no successful backups by that name) xtrabackup will "
+ "return with an error. It is used with the --incremental option.",
+ (uchar*) &opt_ibx_incremental_history_name,
+ (uchar*) &opt_ibx_incremental_history_name, 0, GET_STR,
+ REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"incremental-history-uuid", OPT_INCREMENTAL_HISTORY_UUID,
+ "This option specifies the UUID of the specific history record "
+ "stored in the PERCONA_SCHEMA.xtrabackup_history to base an "
+ "incremental backup on. --incremental-history-name, "
+ "--incremental-basedir and --incremental-lsn. If no valid lsn can be "
+ "found (no success record with that uuid) xtrabackup will return "
+ "with an error. It is used with the --incremental option.",
+ (uchar*) &opt_ibx_incremental_history_uuid,
+ (uchar*) &opt_ibx_incremental_history_uuid, 0, GET_STR,
+ REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"ftwrl-wait-query-type", OPT_LOCK_WAIT_QUERY_TYPE,
+ "This option specifies which types of queries are allowed to complete "
+ "before innobackupex will issue the global lock. Default is all.",
+ (uchar*) &opt_ibx_lock_wait_query_type,
+ (uchar*) &opt_ibx_lock_wait_query_type, &query_type_typelib,
+ GET_ENUM, REQUIRED_ARG, QUERY_TYPE_ALL, 0, 0, 0, 0, 0},
+
+ {"kill-long-query-type", OPT_KILL_LONG_QUERY_TYPE,
+ "This option specifies which types of queries should be killed to "
+ "unblock the global lock. Default is \"all\".",
+ (uchar*) &opt_ibx_kill_long_query_type,
+ (uchar*) &opt_ibx_kill_long_query_type, &query_type_typelib,
+ GET_ENUM, REQUIRED_ARG, QUERY_TYPE_SELECT, 0, 0, 0, 0, 0},
+
+ {"history", OPT_HISTORY,
+ "This option enables the tracking of backup history in the "
+ "PERCONA_SCHEMA.xtrabackup_history table. An optional history "
+ "series name may be specified that will be placed with the history "
+ "record for the current backup being taken.",
+ NULL, NULL, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"include", OPT_INCLUDE,
+ "This option is a regular expression to be matched against table "
+ "names in databasename.tablename format. It is passed directly to "
+ "xtrabackup's --tables option. See the xtrabackup documentation for "
+ "details.",
+ (uchar*) &opt_ibx_include,
+ (uchar*) &opt_ibx_include, 0, GET_STR,
+ REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"databases", OPT_DATABASES,
+ "This option specifies the list of databases that innobackupex should "
+ "back up. The option accepts a string argument or path to file that "
+ "contains the list of databases to back up. The list is of the form "
+ "\"databasename1[.table_name1] databasename2[.table_name2] . . .\". "
+ "If this option is not specified, all databases containing MyISAM and "
+ "InnoDB tables will be backed up. Please make sure that --databases "
+ "contains all of the InnoDB databases and tables, so that all of the "
+ "innodb.frm files are also backed up. In case the list is very long, "
+ "this can be specified in a file, and the full path of the file can "
+ "be specified instead of the list. (See option --tables-file.)",
+ (uchar*) &opt_ibx_databases,
+ (uchar*) &opt_ibx_databases, 0, GET_STR,
+ REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"kill-long-queries-timeout", OPT_KILL_LONG_QUERIES_TIMEOUT,
+ "This option specifies the number of seconds innobackupex waits "
+ "between starting FLUSH TABLES WITH READ LOCK and killing those "
+ "queries that block it. Default is 0 seconds, which means "
+ "innobackupex will not attempt to kill any queries.",
+ (uchar*) &opt_ibx_kill_long_queries_timeout,
+ (uchar*) &opt_ibx_kill_long_queries_timeout, 0, GET_UINT,
+ REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"ftwrl-wait-timeout", OPT_LOCK_WAIT_TIMEOUT,
+ "This option specifies time in seconds that innobackupex should wait "
+ "for queries that would block FTWRL before running it. If there are "
+ "still such queries when the timeout expires, innobackupex terminates "
+ "with an error. Default is 0, in which case innobackupex does not "
+ "wait for queries to complete and starts FTWRL immediately.",
+ (uchar*) &opt_ibx_lock_wait_timeout,
+ (uchar*) &opt_ibx_lock_wait_timeout, 0, GET_UINT,
+ REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"ftwrl-wait-threshold", OPT_LOCK_WAIT_THRESHOLD,
+ "This option specifies the query run time threshold which is used by "
+ "innobackupex to detect long-running queries with a non-zero value "
+ "of --ftwrl-wait-timeout. FTWRL is not started until such "
+ "long-running queries exist. This option has no effect if "
+ "--ftwrl-wait-timeout is 0. Default value is 60 seconds.",
+ (uchar*) &opt_ibx_lock_wait_threshold,
+ (uchar*) &opt_ibx_lock_wait_threshold, 0, GET_UINT,
+ REQUIRED_ARG, 60, 0, 0, 0, 0, 0},
+
+ {"debug-sleep-before-unlock", OPT_DEBUG_SLEEP_BEFORE_UNLOCK,
+ "This is a debug-only option used by the XtraBackup test suite.",
+ (uchar*) &opt_ibx_debug_sleep_before_unlock,
+ (uchar*) &opt_ibx_debug_sleep_before_unlock, 0, GET_UINT,
+ REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"safe-slave-backup-timeout", OPT_SAFE_SLAVE_BACKUP_TIMEOUT,
+ "How many seconds --safe-slave-backup should wait for "
+ "Slave_open_temp_tables to become zero. (default 300)",
+ (uchar*) &opt_ibx_safe_slave_backup_timeout,
+ (uchar*) &opt_ibx_safe_slave_backup_timeout, 0, GET_UINT,
+ REQUIRED_ARG, 300, 0, 0, 0, 0, 0},
+
+
+ /* Following command-line options are actually handled by xtrabackup.
+ We put them here with only purpose for them to showup in
+ innobackupex --help output */
+
+ {"close_files", OPT_CLOSE_FILES, "Do not keep files opened. This "
+ "option is passed directly to xtrabackup. Use at your own risk.",
+ (uchar*) &ibx_xb_close_files, (uchar*) &ibx_xb_close_files, 0,
+ GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"compress", OPT_COMPRESS, "This option instructs xtrabackup to "
+ "compress backup copies of InnoDB data files. It is passed directly "
+ "to the xtrabackup child process. Try 'xtrabackup --help' for more "
+ "details.", (uchar*) &ibx_xtrabackup_compress_alg,
+ (uchar*) &ibx_xtrabackup_compress_alg, 0,
+ GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"compress-threads", OPT_COMPRESS_THREADS,
+ "This option specifies the number of worker threads that will be used "
+ "for parallel compression. It is passed directly to the xtrabackup "
+ "child process. Try 'xtrabackup --help' for more details.",
+ (uchar*) &ibx_xtrabackup_compress_threads,
+ (uchar*) &ibx_xtrabackup_compress_threads,
+ 0, GET_UINT, REQUIRED_ARG, 1, 1, UINT_MAX, 0, 0, 0},
+
+ {"compress-chunk-size", OPT_COMPRESS_CHUNK_SIZE, "Size of working "
+ "buffer(s) for compression threads in bytes. The default value "
+ "is 64K.", (uchar*) &ibx_xtrabackup_compress_chunk_size,
+ (uchar*) &ibx_xtrabackup_compress_chunk_size,
+ 0, GET_ULL, REQUIRED_ARG, (1 << 16), 1024, ULONGLONG_MAX, 0, 0, 0},
+
+ {"export", OPT_EXPORT, "This option is passed directly to xtrabackup's "
+ "--export option. It enables exporting individual tables for import "
+ "into another server. See the xtrabackup documentation for details.",
+ (uchar*) &ibx_xtrabackup_export, (uchar*) &ibx_xtrabackup_export,
+ 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"extra-lsndir", OPT_EXTRA_LSNDIR, "This option specifies the "
+ "directory in which to save an extra copy of the "
+ "\"xtrabackup_checkpoints\" file. The option accepts a string "
+ "argument. It is passed directly to xtrabackup's --extra-lsndir "
+ "option. See the xtrabackup documentation for details.",
+ (uchar*) &ibx_xtrabackup_extra_lsndir,
+ (uchar*) &ibx_xtrabackup_extra_lsndir,
+ 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"incremental-basedir", OPT_INCREMENTAL_BASEDIR, "This option "
+ "specifies the directory containing the full backup that is the base "
+ "dataset for the incremental backup. The option accepts a string "
+ "argument. It is used with the --incremental option.",
+ (uchar*) &ibx_xtrabackup_incremental_basedir,
+ (uchar*) &ibx_xtrabackup_incremental_basedir,
+ 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"incremental-dir", OPT_INCREMENTAL_DIR, "This option specifies the "
+ "directory where the incremental backup will be combined with the "
+ "full backup to make a new full backup. The option accepts a string "
+ "argument. It is used with the --incremental option.",
+ (uchar*) &ibx_xtrabackup_incremental_dir,
+ (uchar*) &ibx_xtrabackup_incremental_dir,
+ 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"incremental-force-scan", OPT_INCREMENTAL_FORCE_SCAN,
+ "This options tells xtrabackup to perform full scan of data files "
+ "for taking an incremental backup even if full changed page bitmap "
+ "data is available to enable the backup without the full scan.",
+ (uchar*)&ibx_xtrabackup_incremental_force_scan,
+ (uchar*)&ibx_xtrabackup_incremental_force_scan, 0, GET_BOOL, NO_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"log-copy-interval", OPT_LOG_COPY_INTERVAL, "This option specifies "
+ "time interval between checks done by log copying thread in "
+ "milliseconds.", (uchar*) &ibx_xtrabackup_log_copy_interval,
+ (uchar*) &ibx_xtrabackup_log_copy_interval,
+ 0, GET_LONG, REQUIRED_ARG, 1000, 0, LONG_MAX, 0, 1, 0},
+
+ {"incremental-lsn", OPT_INCREMENTAL, "This option specifies the log "
+ "sequence number (LSN) to use for the incremental backup. The option "
+ "accepts a string argument. It is used with the --incremental option. "
+ "It is used instead of specifying --incremental-basedir. For "
+ "databases created by MySQL and Percona Server 5.0-series versions, "
+ "specify the LSN as two 32-bit integers in high:low format. For "
+ "databases created in 5.1 and later, specify the LSN as a single "
+ "64-bit integer.",
+ (uchar*) &ibx_xtrabackup_incremental,
+ (uchar*) &ibx_xtrabackup_incremental,
+ 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"parallel", OPT_PARALLEL, "On backup, this option specifies the "
+ "number of threads the xtrabackup child process should use to back "
+ "up files concurrently. The option accepts an integer argument. It "
+ "is passed directly to xtrabackup's --parallel option. See the "
+ "xtrabackup documentation for details.",
+ (uchar*) &ibx_xtrabackup_parallel, (uchar*) &ibx_xtrabackup_parallel,
+ 0, GET_INT, REQUIRED_ARG, 1, 1, INT_MAX, 0, 0, 0},
+
+
+ {"stream", OPT_STREAM, "This option specifies the format in which to "
+ "do the streamed backup. The option accepts a string argument. The "
+ "backup will be done to STDOUT in the specified format. Currently, "
+ "the only supported formats are tar and xbstream. This option is "
+ "passed directly to xtrabackup's --stream option.",
+ (uchar*) &ibx_xtrabackup_stream_str,
+ (uchar*) &ibx_xtrabackup_stream_str, 0, GET_STR,
+ REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"tables-file", OPT_TABLES_FILE, "This option specifies the file in "
+ "which there are a list of names of the form database. The option "
+ "accepts a string argument.table, one per line. The option is passed "
+ "directly to xtrabackup's --tables-file option.",
+ (uchar*) &ibx_xtrabackup_tables_file,
+ (uchar*) &ibx_xtrabackup_tables_file,
+ 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
+
+ {"throttle", OPT_THROTTLE, "This option specifies a number of I/O "
+ "operations (pairs of read+write) per second. It accepts an integer "
+ "argument. It is passed directly to xtrabackup's --throttle option.",
+ (uchar*) &ibx_xtrabackup_throttle, (uchar*) &ibx_xtrabackup_throttle,
+ 0, GET_LONG, REQUIRED_ARG, 0, 0, LONG_MAX, 0, 1, 0},
+
+ {"tmpdir", 't', "This option specifies the location where a temporary "
+ "files will be stored. If the option is not specified, the default is "
+ "to use the value of tmpdir read from the server configuration.",
+ (uchar*) &ibx_opt_mysql_tmpdir,
+ (uchar*) &ibx_opt_mysql_tmpdir, 0, GET_STR, REQUIRED_ARG,
+ 0, 0, 0, 0, 0, 0},
+
+ {"use-memory", OPT_USE_MEMORY, "This option accepts a string argument "
+ "that specifies the amount of memory in bytes for xtrabackup to use "
+ "for crash recovery while preparing a backup. Multiples are supported "
+ "providing the unit (e.g. 1MB, 1GB). It is used only with the option "
+ "--apply-log. It is passed directly to xtrabackup's --use-memory "
+ "option. See the xtrabackup documentation for details.",
+ (uchar*) &ibx_xtrabackup_use_memory,
+ (uchar*) &ibx_xtrabackup_use_memory,
+ 0, GET_LL, REQUIRED_ARG, 100*1024*1024L, 1024*1024L, LONGLONG_MAX, 0,
+ 1024*1024L, 0},
+
+ { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
+};
+
+
+static void usage(void)
+{
+ puts("Open source backup tool for InnoDB and XtraDB\n\
+\n\
+Copyright (C) 2009-2015 Percona LLC and/or its affiliates.\n\
+Portions Copyright (C) 2000, 2011, MySQL AB & Innobase Oy. All Rights Reserved.\n\
+\n\
+This program is free software; you can redistribute it and/or\n\
+modify it under the terms of the GNU General Public License\n\
+as published by the Free Software Foundation version 2\n\
+of the License.\n\
+\n\
+This program is distributed in the hope that it will be useful,\n\
+but WITHOUT ANY WARRANTY; without even the implied warranty of\n\
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n\
+GNU General Public License for more details.\n\
+\n\
+You can download full text of the license on http://www.gnu.org/licenses/gpl-2.0.txt\n\n");
+
+ puts("innobackupex - Non-blocking backup tool for InnoDB, XtraDB and HailDB databases\n\
+\n\
+SYNOPOSIS\n\
+\n\
+innobackupex [--compress] [--compress-threads=NUMBER-OF-THREADS] [--compress-chunk-size=CHUNK-SIZE]\n\
+ [--include=REGEXP] [--user=NAME]\n\
+ [--password=WORD] [--port=PORT] [--socket=SOCKET]\n\
+ [--no-timestamp] [--ibbackup=IBBACKUP-BINARY]\n\
+ [--slave-info] [--galera-info] [--stream=tar|xbstream]\n\
+ [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME]\n\
+ [--databases=LIST] [--no-lock] \n\
+ [--tmpdir=DIRECTORY] [--tables-file=FILE]\n\
+ [--history=NAME]\n\
+ [--incremental] [--incremental-basedir]\n\
+ [--incremental-dir] [--incremental-force-scan] [--incremental-lsn]\n\
+ [--incremental-history-name=NAME] [--incremental-history-uuid=UUID]\n\
+ [--close-files]\n\
+ BACKUP-ROOT-DIR\n\
+\n\
+innobackupex --apply-log [--use-memory=B]\n\
+ [--defaults-file=MY.CNF]\n\
+ [--export] [--ibbackup=IBBACKUP-BINARY]\n\
+ BACKUP-DIR\n\
+\n\
+innobackupex --copy-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME] BACKUP-DIR\n\
+\n\
+innobackupex --move-back [--defaults-file=MY.CNF] [--defaults-group=GROUP-NAME] BACKUP-DIR\n\
+\n\
+innobackupex [--decompress]\n\
+ [--parallel=NUMBER-OF-FORKS] BACKUP-DIR\n\
+\n\
+DESCRIPTION\n\
+\n\
+The first command line above makes a hot backup of a MySQL database.\n\
+By default it creates a backup directory (named by the current date\n\
+ and time) in the given backup root directory. With the --no-timestamp\n\
+option it does not create a time-stamped backup directory, but it puts\n\
+the backup in the given directory (which must not exist). This\n\
+command makes a complete backup of all MyISAM and InnoDB tables and\n\
+indexes in all databases or in all of the databases specified with the\n\
+--databases option. The created backup contains .frm, .MRG, .MYD,\n\
+.MYI, .MAD, .MAI, .TRG, .TRN, .ARM, .ARZ, .CSM, CSV, .opt, .par, and\n\
+InnoDB data and log files. The MY.CNF options file defines the\n\
+location of the database. This command connects to the MySQL server\n\
+using the mysql client program, and runs xtrabackup as a child\n\
+process.\n\
+\n\
+The --apply-log command prepares a backup for starting a MySQL\n\
+server on the backup. This command recovers InnoDB data files as specified\n\
+in BACKUP-DIR/backup-my.cnf using BACKUP-DIR/ib_logfile0,\n\
+and creates new InnoDB log files as specified in BACKUP-DIR/backup-my.cnf.\n\
+The BACKUP-DIR should be the path to a backup directory created by\n\
+xtrabackup. This command runs xtrabackup as a child process, but it does not \n\
+connect to the database server.\n\
+\n\
+The --copy-back command copies data, index, and log files\n\
+from the backup directory back to their original locations.\n\
+The MY.CNF options file defines the original location of the database.\n\
+The BACKUP-DIR is the path to a backup directory created by xtrabackup.\n\
+\n\
+The --move-back command is similar to --copy-back with the only difference that\n\
+it moves files to their original locations rather than copies them. As this\n\
+option removes backup files, it must be used with caution. It may be useful in\n\
+cases when there is not enough free disk space to copy files.\n\
+\n\
+The --decompress command will decompress a backup made\n\
+with the --compress option. The\n\
+--parallel option will allow multiple files to be decompressed\n\
+simultaneously. In order to decompress, the qpress utility MUST be installed\n\
+and accessable within the path. This process will remove the original\n\
+compressed files and leave the results in the same location.\n\
+\n\
+On success the exit code innobackupex is 0. A non-zero exit code \n\
+indicates an error.\n");
+ printf("Usage: [%s [--defaults-file=#] --backup | %s [--defaults-file=#] --prepare] [OPTIONS]\n", my_progname, my_progname);
+ my_print_help(ibx_long_options);
+}
+
+
+static
+my_bool
+ibx_get_one_option(int optid,
+ const struct my_option *opt __attribute__((unused)),
+ char *argument)
+{
+ switch(optid) {
+ case '?':
+ usage();
+ exit(0);
+ break;
+ case 'v':
+ msg("innobackupex version %s %s (%s)\n",
+ MYSQL_SERVER_VERSION,
+ SYSTEM_TYPE, MACHINE_TYPE);
+ exit(0);
+ break;
+ case OPT_HISTORY:
+ if (argument) {
+ opt_ibx_history = argument;
+ } else {
+ opt_ibx_history = "";
+ }
+ break;
+ case OPT_STREAM:
+ if (!strcasecmp(argument, "xbstream"))
+ xtrabackup_stream_fmt = XB_STREAM_FMT_XBSTREAM;
+ else {
+ ibx_msg("Invalid --stream argument: %s\n", argument);
+ return 1;
+ }
+ xtrabackup_stream = TRUE;
+ break;
+ case OPT_COMPRESS:
+ if (argument == NULL)
+ xtrabackup_compress_alg = "quicklz";
+ else if (strcasecmp(argument, "quicklz"))
+ {
+ ibx_msg("Invalid --compress argument: %s\n", argument);
+ return 1;
+ }
+ xtrabackup_compress = TRUE;
+ break;
+ case 'p':
+ if (argument)
+ {
+ char *start = argument;
+ my_free(opt_ibx_password);
+ opt_ibx_password= my_strdup(argument, MYF(MY_FAE));
+ /* Destroy argument */
+ while (*argument)
+ *argument++= 'x';
+ if (*start)
+ start[1]=0 ;
+ }
+ break;
+ }
+ return(0);
+}
+
+bool
+make_backup_dir()
+{
+ time_t t = time(NULL);
+ char buf[100];
+
+ if (!opt_ibx_notimestamp && !ibx_xtrabackup_stream_str) {
+ strftime(buf, sizeof(buf), "%Y-%m-%d_%H-%M-%S", localtime(&t));
+ ut_a(asprintf(&ibx_backup_directory, "%s/%s",
+ ibx_position_arg, buf) != -1);
+ } else {
+ ibx_backup_directory = strdup(ibx_position_arg);
+ }
+
+ if (!directory_exists(ibx_backup_directory, true)) {
+ return(false);
+ }
+
+ return(true);
+}
+
+bool
+ibx_handle_options(int *argc, char ***argv)
+{
+ int i, n_arguments;
+
+ if (handle_options(argc, argv, ibx_long_options, ibx_get_one_option)) {
+ return(false);
+ }
+
+ if (opt_ibx_apply_log) {
+ ibx_mode = IBX_MODE_APPLY_LOG;
+ } else if (opt_ibx_copy_back) {
+ ibx_mode = IBX_MODE_COPY_BACK;
+ } else if (opt_ibx_move_back) {
+ ibx_mode = IBX_MODE_MOVE_BACK;
+ } else if (opt_ibx_decompress) {
+ ibx_mode = IBX_MODE_DECRYPT_DECOMPRESS;
+ } else {
+ ibx_mode = IBX_MODE_BACKUP;
+ }
+
+ /* find and save position argument */
+ i = 0;
+ n_arguments = 0;
+ while (i < *argc) {
+ char *opt = (*argv)[i];
+
+ if (strncmp(opt, "--", 2) != 0
+ && !(strlen(opt) == 2 && opt[0] == '-')) {
+ if (ibx_position_arg != NULL
+ && ibx_position_arg != opt) {
+ ibx_msg("Error: extra argument found %s\n",
+ opt);
+ }
+ ibx_position_arg = opt;
+ ++n_arguments;
+ }
+ ++i;
+ }
+
+ *argc -= n_arguments;
+ if (n_arguments > 1) {
+ return(false);
+ }
+
+ if (ibx_position_arg == NULL) {
+ ibx_msg("Missing argument\n");
+ return(false);
+ }
+
+ /* set argv[0] to be the program name */
+ --(*argv);
+ ++(*argc);
+
+ return(true);
+}
+
+/*********************************************************************//**
+Parse command-line options, connect to MySQL server,
+detect server capabilities, etc.
+@return true on success. */
+bool
+ibx_init()
+{
+ const char *run;
+
+ /*=====================*/
+ xtrabackup_copy_back = opt_ibx_copy_back;
+ xtrabackup_move_back = opt_ibx_move_back;
+ opt_galera_info = opt_ibx_galera_info;
+ opt_slave_info = opt_ibx_slave_info;
+ opt_no_lock = opt_ibx_no_lock;
+ opt_safe_slave_backup = opt_ibx_safe_slave_backup;
+ opt_rsync = opt_ibx_rsync;
+ opt_force_non_empty_dirs = opt_ibx_force_non_empty_dirs;
+ opt_noversioncheck = opt_ibx_noversioncheck;
+ opt_no_backup_locks = opt_ibx_no_backup_locks;
+ opt_decompress = opt_ibx_decompress;
+
+ opt_incremental_history_name = opt_ibx_incremental_history_name;
+ opt_incremental_history_uuid = opt_ibx_incremental_history_uuid;
+
+ opt_user = opt_ibx_user;
+ opt_password = opt_ibx_password;
+ opt_host = opt_ibx_host;
+ opt_defaults_group = opt_ibx_defaults_group;
+ opt_socket = opt_ibx_socket;
+ opt_port = opt_ibx_port;
+
+ opt_lock_wait_query_type = opt_ibx_lock_wait_query_type;
+ opt_kill_long_query_type = opt_ibx_kill_long_query_type;
+
+ opt_kill_long_queries_timeout = opt_ibx_kill_long_queries_timeout;
+ opt_lock_wait_timeout = opt_ibx_lock_wait_timeout;
+ opt_lock_wait_threshold = opt_ibx_lock_wait_threshold;
+ opt_debug_sleep_before_unlock = opt_ibx_debug_sleep_before_unlock;
+ opt_safe_slave_backup_timeout = opt_ibx_safe_slave_backup_timeout;
+
+ opt_history = opt_ibx_history;
+
+ /* setup xtrabackup options */
+ xb_close_files = ibx_xb_close_files;
+ xtrabackup_compress_alg = ibx_xtrabackup_compress_alg;
+ xtrabackup_compress_threads = ibx_xtrabackup_compress_threads;
+ xtrabackup_compress_chunk_size = ibx_xtrabackup_compress_chunk_size;
+ xtrabackup_export = ibx_xtrabackup_export;
+ xtrabackup_extra_lsndir = ibx_xtrabackup_extra_lsndir;
+ xtrabackup_incremental_basedir = ibx_xtrabackup_incremental_basedir;
+ xtrabackup_incremental_dir = ibx_xtrabackup_incremental_dir;
+ xtrabackup_incremental_force_scan =
+ ibx_xtrabackup_incremental_force_scan;
+ xtrabackup_log_copy_interval = ibx_xtrabackup_log_copy_interval;
+ xtrabackup_incremental = ibx_xtrabackup_incremental;
+ xtrabackup_parallel = ibx_xtrabackup_parallel;
+ xtrabackup_stream_str = ibx_xtrabackup_stream_str;
+ xtrabackup_tables_file = ibx_xtrabackup_tables_file;
+ xtrabackup_throttle = ibx_xtrabackup_throttle;
+ opt_mysql_tmpdir = ibx_opt_mysql_tmpdir;
+ xtrabackup_use_memory = ibx_xtrabackup_use_memory;
+
+ if (!opt_ibx_incremental
+ && (xtrabackup_incremental
+ || xtrabackup_incremental_basedir
+ || opt_ibx_incremental_history_name
+ || opt_ibx_incremental_history_uuid)) {
+ ibx_msg("Error: --incremental-lsn, --incremental-basedir, "
+ "--incremental-history-name and "
+ "--incremental-history-uuid require the "
+ "--incremental option.\n");
+ return(false);
+ }
+
+ if (opt_ibx_databases != NULL) {
+ if (is_path_separator(*opt_ibx_databases)) {
+ xtrabackup_databases_file = opt_ibx_databases;
+ } else {
+ xtrabackup_databases = opt_ibx_databases;
+ }
+ }
+
+ /* --tables and --tables-file options are xtrabackup only */
+ ibx_partial_backup = (opt_ibx_include || opt_ibx_databases);
+
+ if (ibx_mode == IBX_MODE_BACKUP) {
+
+ if (!make_backup_dir()) {
+ return(false);
+ }
+ }
+
+ /* --binlog-info is xtrabackup only, so force
+ --binlog-info=ON. i.e. behavior before the feature had been
+ implemented */
+ opt_binlog_info = BINLOG_INFO_ON;
+
+ switch (ibx_mode) {
+ case IBX_MODE_APPLY_LOG:
+ xtrabackup_prepare = TRUE;
+ xtrabackup_target_dir = ibx_position_arg;
+ run = "apply-log";
+ break;
+ case IBX_MODE_BACKUP:
+ xtrabackup_backup = TRUE;
+ xtrabackup_target_dir = ibx_backup_directory;
+ if (opt_ibx_include != NULL) {
+ xtrabackup_tables = opt_ibx_include;
+ }
+ run = "backup";
+ break;
+ case IBX_MODE_COPY_BACK:
+ xtrabackup_copy_back = TRUE;
+ xtrabackup_target_dir = ibx_position_arg;
+ run = "copy-back";
+ break;
+ case IBX_MODE_MOVE_BACK:
+ xtrabackup_move_back = TRUE;
+ xtrabackup_target_dir = ibx_position_arg;
+ run = "move-back";
+ break;
+ case IBX_MODE_DECRYPT_DECOMPRESS:
+ xtrabackup_decrypt_decompress = TRUE;
+ xtrabackup_target_dir = ibx_position_arg;
+ run = "decompress";
+ break;
+ default:
+ ut_error;
+ }
+
+ ibx_msg("Starting the %s operation\n\n"
+ "IMPORTANT: Please check that the %s run completes "
+ "successfully.\n"
+ " At the end of a successful %s run innobackupex\n"
+ " prints \"completed OK!\".\n\n", run, run, run);
+
+
+ return(true);
+}
+
+void
+ibx_cleanup()
+{
+ free(ibx_backup_directory);
+}
diff --git a/extra/mariabackup/innobackupex.h b/extra/mariabackup/innobackupex.h
new file mode 100644
index 00000000000..e2ad9bd2511
--- /dev/null
+++ b/extra/mariabackup/innobackupex.h
@@ -0,0 +1,45 @@
+/******************************************************
+Copyright (c) 2011-2014 Percona LLC and/or its affiliates.
+
+Declarations for innobackupex.cc
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#ifndef INNOBACKUPEX_H
+#define INNOBACKUPEX_H
+
+#define INNOBACKUPEX_BIN_NAME "innobackupex"
+
+enum ibx_mode_t {
+ IBX_MODE_BACKUP,
+ IBX_MODE_APPLY_LOG,
+ IBX_MODE_COPY_BACK,
+ IBX_MODE_MOVE_BACK,
+ IBX_MODE_DECRYPT_DECOMPRESS
+};
+
+extern ibx_mode_t ibx_mode;
+
+bool
+ibx_handle_options(int *argc, char ***argv);
+
+bool
+ibx_init();
+
+void
+ibx_cleanup();
+
+#endif
diff --git a/extra/mariabackup/quicklz/quicklz.c b/extra/mariabackup/quicklz/quicklz.c
new file mode 100644
index 00000000000..3742129023a
--- /dev/null
+++ b/extra/mariabackup/quicklz/quicklz.c
@@ -0,0 +1,848 @@
+// Fast data compression library
+// Copyright (C) 2006-2011 Lasse Mikkel Reinhold
+// lar@quicklz.com
+//
+// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything
+// released into public must be open source) or under a commercial license if such
+// has been acquired (see http://www.quicklz.com/order.html). The commercial license
+// does not cover derived or ported versions created by third parties under GPL.
+
+// 1.5.0 final
+
+#include "quicklz.h"
+
+#if QLZ_VERSION_MAJOR != 1 || QLZ_VERSION_MINOR != 5 || QLZ_VERSION_REVISION != 0
+ #error quicklz.c and quicklz.h have different versions
+#endif
+
+#if (defined(__X86__) || defined(__i386__) || defined(i386) || defined(_M_IX86) || defined(__386__) || defined(__x86_64__) || defined(_M_X64))
+ #define X86X64
+#endif
+
+#define MINOFFSET 2
+#define UNCONDITIONAL_MATCHLEN 6
+#define UNCOMPRESSED_END 4
+#define CWORD_LEN 4
+
+#if QLZ_COMPRESSION_LEVEL == 1 && defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0
+ #define OFFSET_BASE source
+ #define CAST (ui32)(size_t)
+#else
+ #define OFFSET_BASE 0
+ #define CAST
+#endif
+
+int qlz_get_setting(int setting)
+{
+ switch (setting)
+ {
+ case 0: return QLZ_COMPRESSION_LEVEL;
+ case 1: return sizeof(qlz_state_compress);
+ case 2: return sizeof(qlz_state_decompress);
+ case 3: return QLZ_STREAMING_BUFFER;
+#ifdef QLZ_MEMORY_SAFE
+ case 6: return 1;
+#else
+ case 6: return 0;
+#endif
+ case 7: return QLZ_VERSION_MAJOR;
+ case 8: return QLZ_VERSION_MINOR;
+ case 9: return QLZ_VERSION_REVISION;
+ }
+ return -1;
+}
+
+#if QLZ_COMPRESSION_LEVEL == 1
+static int same(const unsigned char *src, size_t n)
+{
+ while(n > 0 && *(src + n) == *src)
+ n--;
+ return n == 0 ? 1 : 0;
+}
+#endif
+
+static void reset_table_compress(qlz_state_compress *state)
+{
+ int i;
+ for(i = 0; i < QLZ_HASH_VALUES; i++)
+ {
+#if QLZ_COMPRESSION_LEVEL == 1
+ state->hash[i].offset = 0;
+#else
+ state->hash_counter[i] = 0;
+#endif
+ }
+}
+
+static void reset_table_decompress(qlz_state_decompress *state)
+{
+ int i;
+ (void)state;
+ (void)i;
+#if QLZ_COMPRESSION_LEVEL == 2
+ for(i = 0; i < QLZ_HASH_VALUES; i++)
+ {
+ state->hash_counter[i] = 0;
+ }
+#endif
+}
+
+static __inline ui32 hash_func(ui32 i)
+{
+#if QLZ_COMPRESSION_LEVEL == 2
+ return ((i >> 9) ^ (i >> 13) ^ i) & (QLZ_HASH_VALUES - 1);
+#else
+ return ((i >> 12) ^ i) & (QLZ_HASH_VALUES - 1);
+#endif
+}
+
+static __inline ui32 fast_read(void const *src, ui32 bytes)
+{
+#ifndef X86X64
+ unsigned char *p = (unsigned char*)src;
+ switch (bytes)
+ {
+ case 4:
+ return(*p | *(p + 1) << 8 | *(p + 2) << 16 | *(p + 3) << 24);
+ case 3:
+ return(*p | *(p + 1) << 8 | *(p + 2) << 16);
+ case 2:
+ return(*p | *(p + 1) << 8);
+ case 1:
+ return(*p);
+ }
+ return 0;
+#else
+ if (bytes >= 1 && bytes <= 4)
+ return *((ui32*)src);
+ else
+ return 0;
+#endif
+}
+
+static __inline ui32 hashat(const unsigned char *src)
+{
+ ui32 fetch, hash;
+ fetch = fast_read(src, 3);
+ hash = hash_func(fetch);
+ return hash;
+}
+
+static __inline void fast_write(ui32 f, void *dst, size_t bytes)
+{
+#ifndef X86X64
+ unsigned char *p = (unsigned char*)dst;
+
+ switch (bytes)
+ {
+ case 4:
+ *p = (unsigned char)f;
+ *(p + 1) = (unsigned char)(f >> 8);
+ *(p + 2) = (unsigned char)(f >> 16);
+ *(p + 3) = (unsigned char)(f >> 24);
+ return;
+ case 3:
+ *p = (unsigned char)f;
+ *(p + 1) = (unsigned char)(f >> 8);
+ *(p + 2) = (unsigned char)(f >> 16);
+ return;
+ case 2:
+ *p = (unsigned char)f;
+ *(p + 1) = (unsigned char)(f >> 8);
+ return;
+ case 1:
+ *p = (unsigned char)f;
+ return;
+ }
+#else
+ switch (bytes)
+ {
+ case 4:
+ *((ui32*)dst) = f;
+ return;
+ case 3:
+ *((ui32*)dst) = f;
+ return;
+ case 2:
+ *((ui16 *)dst) = (ui16)f;
+ return;
+ case 1:
+ *((unsigned char*)dst) = (unsigned char)f;
+ return;
+ }
+#endif
+}
+
+
+size_t qlz_size_decompressed(const char *source)
+{
+ ui32 n, r;
+ n = (((*source) & 2) == 2) ? 4 : 1;
+ r = fast_read(source + 1 + n, n);
+ r = r & (0xffffffff >> ((4 - n)*8));
+ return r;
+}
+
+size_t qlz_size_compressed(const char *source)
+{
+ ui32 n, r;
+ n = (((*source) & 2) == 2) ? 4 : 1;
+ r = fast_read(source + 1, n);
+ r = r & (0xffffffff >> ((4 - n)*8));
+ return r;
+}
+
+size_t qlz_size_header(const char *source)
+{
+ size_t n = 2*((((*source) & 2) == 2) ? 4 : 1) + 1;
+ return n;
+}
+
+
+static __inline void memcpy_up(unsigned char *dst, const unsigned char *src, ui32 n)
+{
+ // Caution if modifying memcpy_up! Overlap of dst and src must be special handled.
+#ifndef X86X64
+ unsigned char *end = dst + n;
+ while(dst < end)
+ {
+ *dst = *src;
+ dst++;
+ src++;
+ }
+#else
+ ui32 f = 0;
+ do
+ {
+ *(ui32 *)(dst + f) = *(ui32 *)(src + f);
+ f += MINOFFSET + 1;
+ }
+ while (f < n);
+#endif
+}
+
+static __inline void update_hash(qlz_state_decompress *state, const unsigned char *s)
+{
+#if QLZ_COMPRESSION_LEVEL == 1
+ ui32 hash;
+ hash = hashat(s);
+ state->hash[hash].offset = s;
+ state->hash_counter[hash] = 1;
+#elif QLZ_COMPRESSION_LEVEL == 2
+ ui32 hash;
+ unsigned char c;
+ hash = hashat(s);
+ c = state->hash_counter[hash];
+ state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = s;
+ c++;
+ state->hash_counter[hash] = c;
+#endif
+ (void)state;
+ (void)s;
+}
+
+#if QLZ_COMPRESSION_LEVEL <= 2
+static void update_hash_upto(qlz_state_decompress *state, unsigned char **lh, const unsigned char *max)
+{
+ while(*lh < max)
+ {
+ (*lh)++;
+ update_hash(state, *lh);
+ }
+}
+#endif
+
+static size_t qlz_compress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_compress *state)
+{
+ const unsigned char *last_byte = source + size - 1;
+ const unsigned char *src = source;
+ unsigned char *cword_ptr = destination;
+ unsigned char *dst = destination + CWORD_LEN;
+ ui32 cword_val = 1U << 31;
+ const unsigned char *last_matchstart = last_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END;
+ ui32 fetch = 0;
+ unsigned int lits = 0;
+
+ (void) lits;
+
+ if(src <= last_matchstart)
+ fetch = fast_read(src, 3);
+
+ while(src <= last_matchstart)
+ {
+ if ((cword_val & 1) == 1)
+ {
+ // store uncompressed if compression ratio is too low
+ if (src > source + (size >> 1) && dst - destination > src - source - ((src - source) >> 5))
+ return 0;
+
+ fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
+
+ cword_ptr = dst;
+ dst += CWORD_LEN;
+ cword_val = 1U << 31;
+ fetch = fast_read(src, 3);
+ }
+#if QLZ_COMPRESSION_LEVEL == 1
+ {
+ const unsigned char *o;
+ ui32 hash, cached;
+
+ hash = hash_func(fetch);
+ cached = fetch ^ state->hash[hash].cache;
+ state->hash[hash].cache = fetch;
+
+ o = state->hash[hash].offset + OFFSET_BASE;
+ state->hash[hash].offset = CAST(src - OFFSET_BASE);
+
+#ifdef X86X64
+ if ((cached & 0xffffff) == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6))))
+ {
+ if(cached != 0)
+ {
+#else
+ if (cached == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6))))
+ {
+ if (*(o + 3) != *(src + 3))
+ {
+#endif
+ hash <<= 4;
+ cword_val = (cword_val >> 1) | (1U << 31);
+ fast_write((3 - 2) | hash, dst, 2);
+ src += 3;
+ dst += 2;
+ }
+ else
+ {
+ const unsigned char *old_src = src;
+ size_t matchlen;
+ hash <<= 4;
+
+ cword_val = (cword_val >> 1) | (1U << 31);
+ src += 4;
+
+ if(*(o + (src - old_src)) == *src)
+ {
+ src++;
+ if(*(o + (src - old_src)) == *src)
+ {
+ size_t q = last_byte - UNCOMPRESSED_END - (src - 5) + 1;
+ size_t remaining = q > 255 ? 255 : q;
+ src++;
+ while(*(o + (src - old_src)) == *src && (size_t)(src - old_src) < remaining)
+ src++;
+ }
+ }
+
+ matchlen = src - old_src;
+ if (matchlen < 18)
+ {
+ fast_write((ui32)(matchlen - 2) | hash, dst, 2);
+ dst += 2;
+ }
+ else
+ {
+ fast_write((ui32)(matchlen << 16) | hash, dst, 3);
+ dst += 3;
+ }
+ }
+ fetch = fast_read(src, 3);
+ lits = 0;
+ }
+ else
+ {
+ lits++;
+ *dst = *src;
+ src++;
+ dst++;
+ cword_val = (cword_val >> 1);
+#ifdef X86X64
+ fetch = fast_read(src, 3);
+#else
+ fetch = (fetch >> 8 & 0xffff) | (*(src + 2) << 16);
+#endif
+ }
+ }
+#elif QLZ_COMPRESSION_LEVEL >= 2
+ {
+ const unsigned char *o, *offset2;
+ ui32 hash, matchlen, k, m, best_k = 0;
+ unsigned char c;
+ size_t remaining = (last_byte - UNCOMPRESSED_END - src + 1) > 255 ? 255 : (last_byte - UNCOMPRESSED_END - src + 1);
+ (void)best_k;
+
+
+ //hash = hashat(src);
+ fetch = fast_read(src, 3);
+ hash = hash_func(fetch);
+
+ c = state->hash_counter[hash];
+
+ offset2 = state->hash[hash].offset[0];
+ if(offset2 < src - MINOFFSET && c > 0 && ((fast_read(offset2, 3) ^ fetch) & 0xffffff) == 0)
+ {
+ matchlen = 3;
+ if(*(offset2 + matchlen) == *(src + matchlen))
+ {
+ matchlen = 4;
+ while(*(offset2 + matchlen) == *(src + matchlen) && matchlen < remaining)
+ matchlen++;
+ }
+ }
+ else
+ matchlen = 0;
+ for(k = 1; k < QLZ_POINTERS && c > k; k++)
+ {
+ o = state->hash[hash].offset[k];
+#if QLZ_COMPRESSION_LEVEL == 3
+ if(((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET)
+#elif QLZ_COMPRESSION_LEVEL == 2
+ if(*(src + matchlen) == *(o + matchlen) && ((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET)
+#endif
+ {
+ m = 3;
+ while(*(o + m) == *(src + m) && m < remaining)
+ m++;
+#if QLZ_COMPRESSION_LEVEL == 3
+ if ((m > matchlen) || (m == matchlen && o > offset2))
+#elif QLZ_COMPRESSION_LEVEL == 2
+ if (m > matchlen)
+#endif
+ {
+ offset2 = o;
+ matchlen = m;
+ best_k = k;
+ }
+ }
+ }
+ o = offset2;
+ state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src;
+ c++;
+ state->hash_counter[hash] = c;
+
+#if QLZ_COMPRESSION_LEVEL == 3
+ if(matchlen > 2 && src - o < 131071)
+ {
+ ui32 u;
+ size_t offset = src - o;
+
+ for(u = 1; u < matchlen; u++)
+ {
+ hash = hashat(src + u);
+ c = state->hash_counter[hash]++;
+ state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src + u;
+ }
+
+ cword_val = (cword_val >> 1) | (1U << 31);
+ src += matchlen;
+
+ if(matchlen == 3 && offset <= 63)
+ {
+ *dst = (unsigned char)(offset << 2);
+ dst++;
+ }
+ else if (matchlen == 3 && offset <= 16383)
+ {
+ ui32 f = (ui32)((offset << 2) | 1);
+ fast_write(f, dst, 2);
+ dst += 2;
+ }
+ else if (matchlen <= 18 && offset <= 1023)
+ {
+ ui32 f = ((matchlen - 3) << 2) | ((ui32)offset << 6) | 2;
+ fast_write(f, dst, 2);
+ dst += 2;
+ }
+
+ else if(matchlen <= 33)
+ {
+ ui32 f = ((matchlen - 2) << 2) | ((ui32)offset << 7) | 3;
+ fast_write(f, dst, 3);
+ dst += 3;
+ }
+ else
+ {
+ ui32 f = ((matchlen - 3) << 7) | ((ui32)offset << 15) | 3;
+ fast_write(f, dst, 4);
+ dst += 4;
+ }
+ }
+ else
+ {
+ *dst = *src;
+ src++;
+ dst++;
+ cword_val = (cword_val >> 1);
+ }
+#elif QLZ_COMPRESSION_LEVEL == 2
+
+ if(matchlen > 2)
+ {
+ cword_val = (cword_val >> 1) | (1U << 31);
+ src += matchlen;
+
+ if (matchlen < 10)
+ {
+ ui32 f = best_k | ((matchlen - 2) << 2) | (hash << 5);
+ fast_write(f, dst, 2);
+ dst += 2;
+ }
+ else
+ {
+ ui32 f = best_k | (matchlen << 16) | (hash << 5);
+ fast_write(f, dst, 3);
+ dst += 3;
+ }
+ }
+ else
+ {
+ *dst = *src;
+ src++;
+ dst++;
+ cword_val = (cword_val >> 1);
+ }
+#endif
+ }
+#endif
+ }
+ while (src <= last_byte)
+ {
+ if ((cword_val & 1) == 1)
+ {
+ fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
+ cword_ptr = dst;
+ dst += CWORD_LEN;
+ cword_val = 1U << 31;
+ }
+#if QLZ_COMPRESSION_LEVEL < 3
+ if (src <= last_byte - 3)
+ {
+#if QLZ_COMPRESSION_LEVEL == 1
+ ui32 hash, fetch;
+ fetch = fast_read(src, 3);
+ hash = hash_func(fetch);
+ state->hash[hash].offset = CAST(src - OFFSET_BASE);
+ state->hash[hash].cache = fetch;
+#elif QLZ_COMPRESSION_LEVEL == 2
+ ui32 hash;
+ unsigned char c;
+ hash = hashat(src);
+ c = state->hash_counter[hash];
+ state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src;
+ c++;
+ state->hash_counter[hash] = c;
+#endif
+ }
+#endif
+ *dst = *src;
+ src++;
+ dst++;
+ cword_val = (cword_val >> 1);
+ }
+
+ while((cword_val & 1) != 1)
+ cword_val = (cword_val >> 1);
+
+ fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN);
+
+ // min. size must be 9 bytes so that the qlz_size functions can take 9 bytes as argument
+ return dst - destination < 9 ? 9 : dst - destination;
+}
+
+static size_t qlz_decompress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_decompress *state, const unsigned char *history)
+{
+ const unsigned char *src = source + qlz_size_header((const char *)source);
+ unsigned char *dst = destination;
+ const unsigned char *last_destination_byte = destination + size - 1;
+ ui32 cword_val = 1;
+ const unsigned char *last_matchstart = last_destination_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END;
+ unsigned char *last_hashed = destination - 1;
+ const unsigned char *last_source_byte = source + qlz_size_compressed((const char *)source) - 1;
+ static const ui32 bitlut[16] = {4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0};
+
+ (void) last_source_byte;
+ (void) last_hashed;
+ (void) state;
+ (void) history;
+
+ for(;;)
+ {
+ ui32 fetch;
+
+ if (cword_val == 1)
+ {
+#ifdef QLZ_MEMORY_SAFE
+ if(src + CWORD_LEN - 1 > last_source_byte)
+ return 0;
+#endif
+ cword_val = fast_read(src, CWORD_LEN);
+ src += CWORD_LEN;
+ }
+
+#ifdef QLZ_MEMORY_SAFE
+ if(src + 4 - 1 > last_source_byte)
+ return 0;
+#endif
+
+ fetch = fast_read(src, 4);
+
+ if ((cword_val & 1) == 1)
+ {
+ ui32 matchlen;
+ const unsigned char *offset2;
+
+#if QLZ_COMPRESSION_LEVEL == 1
+ ui32 hash;
+ cword_val = cword_val >> 1;
+ hash = (fetch >> 4) & 0xfff;
+ offset2 = (const unsigned char *)(size_t)state->hash[hash].offset;
+
+ if((fetch & 0xf) != 0)
+ {
+ matchlen = (fetch & 0xf) + 2;
+ src += 2;
+ }
+ else
+ {
+ matchlen = *(src + 2);
+ src += 3;
+ }
+
+#elif QLZ_COMPRESSION_LEVEL == 2
+ ui32 hash;
+ unsigned char c;
+ cword_val = cword_val >> 1;
+ hash = (fetch >> 5) & 0x7ff;
+ c = (unsigned char)(fetch & 0x3);
+ offset2 = state->hash[hash].offset[c];
+
+ if((fetch & (28)) != 0)
+ {
+ matchlen = ((fetch >> 2) & 0x7) + 2;
+ src += 2;
+ }
+ else
+ {
+ matchlen = *(src + 2);
+ src += 3;
+ }
+
+#elif QLZ_COMPRESSION_LEVEL == 3
+ ui32 offset;
+ cword_val = cword_val >> 1;
+ if ((fetch & 3) == 0)
+ {
+ offset = (fetch & 0xff) >> 2;
+ matchlen = 3;
+ src++;
+ }
+ else if ((fetch & 2) == 0)
+ {
+ offset = (fetch & 0xffff) >> 2;
+ matchlen = 3;
+ src += 2;
+ }
+ else if ((fetch & 1) == 0)
+ {
+ offset = (fetch & 0xffff) >> 6;
+ matchlen = ((fetch >> 2) & 15) + 3;
+ src += 2;
+ }
+ else if ((fetch & 127) != 3)
+ {
+ offset = (fetch >> 7) & 0x1ffff;
+ matchlen = ((fetch >> 2) & 0x1f) + 2;
+ src += 3;
+ }
+ else
+ {
+ offset = (fetch >> 15);
+ matchlen = ((fetch >> 7) & 255) + 3;
+ src += 4;
+ }
+
+ offset2 = dst - offset;
+#endif
+
+#ifdef QLZ_MEMORY_SAFE
+ if(offset2 < history || offset2 > dst - MINOFFSET - 1)
+ return 0;
+
+ if(matchlen > (ui32)(last_destination_byte - dst - UNCOMPRESSED_END + 1))
+ return 0;
+#endif
+
+ memcpy_up(dst, offset2, matchlen);
+ dst += matchlen;
+
+#if QLZ_COMPRESSION_LEVEL <= 2
+ update_hash_upto(state, &last_hashed, dst - matchlen);
+ last_hashed = dst - 1;
+#endif
+ }
+ else
+ {
+ if (dst < last_matchstart)
+ {
+ unsigned int n = bitlut[cword_val & 0xf];
+#ifdef X86X64
+ *(ui32 *)dst = *(ui32 *)src;
+#else
+ memcpy_up(dst, src, 4);
+#endif
+ cword_val = cword_val >> n;
+ dst += n;
+ src += n;
+#if QLZ_COMPRESSION_LEVEL <= 2
+ update_hash_upto(state, &last_hashed, dst - 3);
+#endif
+ }
+ else
+ {
+ while(dst <= last_destination_byte)
+ {
+ if (cword_val == 1)
+ {
+ src += CWORD_LEN;
+ cword_val = 1U << 31;
+ }
+#ifdef QLZ_MEMORY_SAFE
+ if(src >= last_source_byte + 1)
+ return 0;
+#endif
+ *dst = *src;
+ dst++;
+ src++;
+ cword_val = cword_val >> 1;
+ }
+
+#if QLZ_COMPRESSION_LEVEL <= 2
+ update_hash_upto(state, &last_hashed, last_destination_byte - 3); // todo, use constant
+#endif
+ return size;
+ }
+
+ }
+ }
+}
+
+size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state)
+{
+ size_t r;
+ ui32 compressed;
+ size_t base;
+
+ if(size == 0 || size > 0xffffffff - 400)
+ return 0;
+
+ if(size < 216)
+ base = 3;
+ else
+ base = 9;
+
+#if QLZ_STREAMING_BUFFER > 0
+ if (state->stream_counter + size - 1 >= QLZ_STREAMING_BUFFER)
+#endif
+ {
+ reset_table_compress(state);
+ r = base + qlz_compress_core((const unsigned char *)source, (unsigned char*)destination + base, size, state);
+#if QLZ_STREAMING_BUFFER > 0
+ reset_table_compress(state);
+#endif
+ if(r == base)
+ {
+ memcpy(destination + base, source, size);
+ r = size + base;
+ compressed = 0;
+ }
+ else
+ {
+ compressed = 1;
+ }
+ state->stream_counter = 0;
+ }
+#if QLZ_STREAMING_BUFFER > 0
+ else
+ {
+ unsigned char *src = state->stream_buffer + state->stream_counter;
+
+ memcpy(src, source, size);
+ r = base + qlz_compress_core(src, (unsigned char*)destination + base, size, state);
+
+ if(r == base)
+ {
+ memcpy(destination + base, src, size);
+ r = size + base;
+ compressed = 0;
+ reset_table_compress(state);
+ }
+ else
+ {
+ compressed = 1;
+ }
+ state->stream_counter += size;
+ }
+#endif
+ if(base == 3)
+ {
+ *destination = (unsigned char)(0 | compressed);
+ *(destination + 1) = (unsigned char)r;
+ *(destination + 2) = (unsigned char)size;
+ }
+ else
+ {
+ *destination = (unsigned char)(2 | compressed);
+ fast_write((ui32)r, destination + 1, 4);
+ fast_write((ui32)size, destination + 5, 4);
+ }
+
+ *destination |= (QLZ_COMPRESSION_LEVEL << 2);
+ *destination |= (1 << 6);
+ *destination |= ((QLZ_STREAMING_BUFFER == 0 ? 0 : (QLZ_STREAMING_BUFFER == 100000 ? 1 : (QLZ_STREAMING_BUFFER == 1000000 ? 2 : 3))) << 4);
+
+// 76543210
+// 01SSLLHC
+
+ return r;
+}
+
+size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state)
+{
+ size_t dsiz = qlz_size_decompressed(source);
+
+#if QLZ_STREAMING_BUFFER > 0
+ if (state->stream_counter + qlz_size_decompressed(source) - 1 >= QLZ_STREAMING_BUFFER)
+#endif
+ {
+ if((*source & 1) == 1)
+ {
+ reset_table_decompress(state);
+ dsiz = qlz_decompress_core((const unsigned char *)source, (unsigned char *)destination, dsiz, state, (const unsigned char *)destination);
+ }
+ else
+ {
+ memcpy(destination, source + qlz_size_header(source), dsiz);
+ }
+ state->stream_counter = 0;
+ reset_table_decompress(state);
+ }
+#if QLZ_STREAMING_BUFFER > 0
+ else
+ {
+ unsigned char *dst = state->stream_buffer + state->stream_counter;
+ if((*source & 1) == 1)
+ {
+ dsiz = qlz_decompress_core((const unsigned char *)source, dst, dsiz, state, (const unsigned char *)state->stream_buffer);
+ }
+ else
+ {
+ memcpy(dst, source + qlz_size_header(source), dsiz);
+ reset_table_decompress(state);
+ }
+ memcpy(destination, dst, dsiz);
+ state->stream_counter += dsiz;
+ }
+#endif
+ return dsiz;
+}
+
diff --git a/extra/mariabackup/quicklz/quicklz.h b/extra/mariabackup/quicklz/quicklz.h
new file mode 100644
index 00000000000..6ffe00f3a91
--- /dev/null
+++ b/extra/mariabackup/quicklz/quicklz.h
@@ -0,0 +1,144 @@
+#ifndef QLZ_HEADER
+#define QLZ_HEADER
+
+// Fast data compression library
+// Copyright (C) 2006-2011 Lasse Mikkel Reinhold
+// lar@quicklz.com
+//
+// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything
+// released into public must be open source) or under a commercial license if such
+// has been acquired (see http://www.quicklz.com/order.html). The commercial license
+// does not cover derived or ported versions created by third parties under GPL.
+
+// You can edit following user settings. Data must be decompressed with the same
+// setting of QLZ_COMPRESSION_LEVEL and QLZ_STREAMING_BUFFER as it was compressed
+// (see manual). If QLZ_STREAMING_BUFFER > 0, scratch buffers must be initially
+// zeroed out (see manual). First #ifndef makes it possible to define settings from
+// the outside like the compiler command line.
+
+// 1.5.0 final
+
+#ifndef QLZ_COMPRESSION_LEVEL
+ #define QLZ_COMPRESSION_LEVEL 1
+ //#define QLZ_COMPRESSION_LEVEL 2
+ //#define QLZ_COMPRESSION_LEVEL 3
+
+ #define QLZ_STREAMING_BUFFER 0
+ //#define QLZ_STREAMING_BUFFER 100000
+ //#define QLZ_STREAMING_BUFFER 1000000
+
+ //#define QLZ_MEMORY_SAFE
+#endif
+
+#define QLZ_VERSION_MAJOR 1
+#define QLZ_VERSION_MINOR 5
+#define QLZ_VERSION_REVISION 0
+
+// Using size_t, memset() and memcpy()
+#include
+
+// Verify compression level
+#if QLZ_COMPRESSION_LEVEL != 1 && QLZ_COMPRESSION_LEVEL != 2 && QLZ_COMPRESSION_LEVEL != 3
+#error QLZ_COMPRESSION_LEVEL must be 1, 2 or 3
+#endif
+
+typedef unsigned int ui32;
+typedef unsigned short int ui16;
+
+// Decrease QLZ_POINTERS for level 3 to increase compression speed. Do not touch any other values!
+#if QLZ_COMPRESSION_LEVEL == 1
+#define QLZ_POINTERS 1
+#define QLZ_HASH_VALUES 4096
+#elif QLZ_COMPRESSION_LEVEL == 2
+#define QLZ_POINTERS 4
+#define QLZ_HASH_VALUES 2048
+#elif QLZ_COMPRESSION_LEVEL == 3
+#define QLZ_POINTERS 16
+#define QLZ_HASH_VALUES 4096
+#endif
+
+// Detect if pointer size is 64-bit. It's not fatal if some 64-bit target is not detected because this is only for adding an optional 64-bit optimization.
+#if defined _LP64 || defined __LP64__ || defined __64BIT__ || _ADDR64 || defined _WIN64 || defined __arch64__ || __WORDSIZE == 64 || (defined __sparc && defined __sparcv9) || defined __x86_64 || defined __amd64 || defined __x86_64__ || defined _M_X64 || defined _M_IA64 || defined __ia64 || defined __IA64__
+ #define QLZ_PTR_64
+#endif
+
+// hash entry
+typedef struct
+{
+#if QLZ_COMPRESSION_LEVEL == 1
+ ui32 cache;
+#if defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0
+ unsigned int offset;
+#else
+ const unsigned char *offset;
+#endif
+#else
+ const unsigned char *offset[QLZ_POINTERS];
+#endif
+
+} qlz_hash_compress;
+
+typedef struct
+{
+#if QLZ_COMPRESSION_LEVEL == 1
+ const unsigned char *offset;
+#else
+ const unsigned char *offset[QLZ_POINTERS];
+#endif
+} qlz_hash_decompress;
+
+
+// states
+typedef struct
+{
+ #if QLZ_STREAMING_BUFFER > 0
+ unsigned char stream_buffer[QLZ_STREAMING_BUFFER];
+ #endif
+ size_t stream_counter;
+ qlz_hash_compress hash[QLZ_HASH_VALUES];
+ unsigned char hash_counter[QLZ_HASH_VALUES];
+} qlz_state_compress;
+
+
+#if QLZ_COMPRESSION_LEVEL == 1 || QLZ_COMPRESSION_LEVEL == 2
+ typedef struct
+ {
+#if QLZ_STREAMING_BUFFER > 0
+ unsigned char stream_buffer[QLZ_STREAMING_BUFFER];
+#endif
+ qlz_hash_decompress hash[QLZ_HASH_VALUES];
+ unsigned char hash_counter[QLZ_HASH_VALUES];
+ size_t stream_counter;
+ } qlz_state_decompress;
+#elif QLZ_COMPRESSION_LEVEL == 3
+ typedef struct
+ {
+#if QLZ_STREAMING_BUFFER > 0
+ unsigned char stream_buffer[QLZ_STREAMING_BUFFER];
+#endif
+#if QLZ_COMPRESSION_LEVEL <= 2
+ qlz_hash_decompress hash[QLZ_HASH_VALUES];
+#endif
+ size_t stream_counter;
+ } qlz_state_decompress;
+#endif
+
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+// Public functions of QuickLZ
+size_t qlz_size_decompressed(const char *source);
+size_t qlz_size_compressed(const char *source);
+size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state);
+size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state);
+int qlz_get_setting(int setting);
+size_t qlz_size_header(const char *source);
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif
+
diff --git a/extra/mariabackup/read_filt.cc b/extra/mariabackup/read_filt.cc
new file mode 100644
index 00000000000..a48591abf29
--- /dev/null
+++ b/extra/mariabackup/read_filt.cc
@@ -0,0 +1,207 @@
+/******************************************************
+XtraBackup: hot backup tool for InnoDB
+(c) 2009-2012 Percona Inc.
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+/* Data file read filter implementation */
+
+#include "read_filt.h"
+#include "common.h"
+#include "fil_cur.h"
+#include "xtrabackup.h"
+
+/****************************************************************//**
+Perform read filter context initialization that is common to all read
+filters. */
+static
+void
+common_init(
+/*========*/
+ xb_read_filt_ctxt_t* ctxt, /*!offset = 0;
+ ctxt->data_file_size = cursor->statinfo.st_size;
+ ctxt->buffer_capacity = cursor->buf_size;
+ ctxt->page_size = cursor->page_size;
+}
+
+/****************************************************************//**
+Initialize the pass-through read filter. */
+static
+void
+rf_pass_through_init(
+/*=================*/
+ xb_read_filt_ctxt_t* ctxt, /*!offset;
+ *read_batch_len = ctxt->data_file_size - ctxt->offset;
+
+ if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) {
+ *read_batch_len = ctxt->buffer_capacity;
+ }
+
+ ctxt->offset += *read_batch_len;
+}
+
+/****************************************************************//**
+Deinitialize the pass-through read filter. */
+static
+void
+rf_pass_through_deinit(
+/*===================*/
+ xb_read_filt_ctxt_t* ctxt __attribute__((unused)))
+ /*!bitmap_range = xb_page_bitmap_range_init(changed_page_bitmap,
+ space_id);
+ ctxt->filter_batch_end = 0;
+}
+
+/****************************************************************//**
+Get the next batch of pages for the bitmap read filter. */
+static
+void
+rf_bitmap_get_next_batch(
+/*=====================*/
+ xb_read_filt_ctxt_t* ctxt, /*!page_size.physical();
+
+ start_page_id = (ulint)(ctxt->offset / page_size);
+
+ xb_a (ctxt->offset % page_size == 0);
+
+ if (start_page_id == ctxt->filter_batch_end) {
+
+ /* Used up all the previous bitmap range, get some more */
+ ulint next_page_id;
+
+ /* Find the next changed page using the bitmap */
+ next_page_id = xb_page_bitmap_range_get_next_bit
+ (ctxt->bitmap_range, TRUE);
+
+ if (next_page_id == ULINT_UNDEFINED) {
+ *read_batch_len = 0;
+ return;
+ }
+
+ ctxt->offset = next_page_id * page_size;
+
+ /* Find the end of the current changed page block by searching
+ for the next cleared bitmap bit */
+ ctxt->filter_batch_end
+ = xb_page_bitmap_range_get_next_bit(ctxt->bitmap_range,
+ FALSE);
+ xb_a(next_page_id < ctxt->filter_batch_end);
+ }
+
+ *read_batch_start = ctxt->offset;
+ if (ctxt->filter_batch_end == ULINT_UNDEFINED) {
+ /* No more cleared bits in the bitmap, need to copy all the
+ remaining pages. */
+ *read_batch_len = ctxt->data_file_size - ctxt->offset;
+ } else {
+ *read_batch_len = ctxt->filter_batch_end * page_size
+ - ctxt->offset;
+ }
+
+ /* If the page block is larger than the buffer capacity, limit it to
+ buffer capacity. The subsequent invocations will continue returning
+ the current block in buffer-sized pieces until ctxt->filter_batch_end
+ is reached, trigerring the next bitmap query. */
+ if (*read_batch_len > (ib_int64_t)ctxt->buffer_capacity) {
+ *read_batch_len = ctxt->buffer_capacity;
+ }
+
+ ctxt->offset += *read_batch_len;
+ xb_a (ctxt->offset % page_size == 0);
+ xb_a (*read_batch_start % page_size == 0);
+ xb_a (*read_batch_len % page_size == 0);
+}
+
+/****************************************************************//**
+Deinitialize the changed page bitmap-based read filter. */
+static
+void
+rf_bitmap_deinit(
+/*=============*/
+ xb_read_filt_ctxt_t* ctxt) /*!bitmap_range);
+}
+
+/* The pass-through read filter */
+xb_read_filt_t rf_pass_through = {
+ &rf_pass_through_init,
+ &rf_pass_through_get_next_batch,
+ &rf_pass_through_deinit
+};
+
+/* The changed page bitmap-based read filter */
+xb_read_filt_t rf_bitmap = {
+ &rf_bitmap_init,
+ &rf_bitmap_get_next_batch,
+ &rf_bitmap_deinit
+};
diff --git a/extra/mariabackup/read_filt.h b/extra/mariabackup/read_filt.h
new file mode 100644
index 00000000000..cebc714eed8
--- /dev/null
+++ b/extra/mariabackup/read_filt.h
@@ -0,0 +1,66 @@
+/******************************************************
+XtraBackup: hot backup tool for InnoDB
+(c) 2009-2012 Percona Inc.
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+/* Data file read filter interface */
+
+#ifndef XB_READ_FILT_H
+#define XB_READ_FILT_H
+
+#include "changed_page_bitmap.h"
+
+typedef ulint space_id_t;
+
+struct xb_fil_cur_t;
+
+/* The read filter context */
+struct xb_read_filt_ctxt_t {
+ ib_int64_t offset; /*!< current file offset */
+ ib_int64_t data_file_size; /*!< data file size */
+ size_t buffer_capacity;/*!< read buffer capacity */
+ space_id_t space_id; /*!< space id */
+ /* The following fields used only in bitmap filter */
+ /* Move these to union if any other filters are added in future */
+ xb_page_bitmap_range *bitmap_range; /*!< changed page bitmap range
+ iterator for space_id */
+ page_size_t page_size; /*!< page size */
+ ulint filter_batch_end;/*!< the ending page id of the
+ current changed page block in
+ the bitmap */
+ /** TODO: remove this default constructor */
+ xb_read_filt_ctxt_t() : page_size(0) {}
+};
+
+/* The read filter */
+struct xb_read_filt_t {
+ void (*init)(xb_read_filt_ctxt_t* ctxt,
+ const xb_fil_cur_t* cursor,
+ ulint space_id);
+ void (*get_next_batch)(xb_read_filt_ctxt_t* ctxt,
+ ib_int64_t* read_batch_start,
+ ib_int64_t* read_batch_len);
+ void (*deinit)(xb_read_filt_ctxt_t* ctxt);
+};
+
+extern xb_read_filt_t rf_pass_through;
+extern xb_read_filt_t rf_bitmap;
+
+#endif
diff --git a/extra/mariabackup/write_filt.cc b/extra/mariabackup/write_filt.cc
new file mode 100644
index 00000000000..a0633818405
--- /dev/null
+++ b/extra/mariabackup/write_filt.cc
@@ -0,0 +1,217 @@
+/******************************************************
+XtraBackup: hot backup tool for InnoDB
+(c) 2009-2013 Percona LLC and/or its affiliates.
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+/* Page write filters implementation */
+
+#include
+#include "common.h"
+#include "write_filt.h"
+#include "fil_cur.h"
+#include "xtrabackup.h"
+
+/************************************************************************
+Write-through page write filter. */
+static my_bool wf_wt_init(xb_write_filt_ctxt_t *ctxt, char *dst_name,
+ xb_fil_cur_t *cursor);
+static my_bool wf_wt_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile);
+
+xb_write_filt_t wf_write_through = {
+ &wf_wt_init,
+ &wf_wt_process,
+ NULL,
+ NULL
+};
+
+/************************************************************************
+Incremental page write filter. */
+static my_bool wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name,
+ xb_fil_cur_t *cursor);
+static my_bool wf_incremental_process(xb_write_filt_ctxt_t *ctxt,
+ ds_file_t *dstfile);
+static my_bool wf_incremental_finalize(xb_write_filt_ctxt_t *ctxt,
+ ds_file_t *dstfile);
+static void wf_incremental_deinit(xb_write_filt_ctxt_t *ctxt);
+
+xb_write_filt_t wf_incremental = {
+ &wf_incremental_init,
+ &wf_incremental_process,
+ &wf_incremental_finalize,
+ &wf_incremental_deinit
+};
+
+/************************************************************************
+Initialize incremental page write filter.
+
+@return TRUE on success, FALSE on error. */
+static my_bool
+wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name,
+ xb_fil_cur_t *cursor)
+{
+ char meta_name[FN_REFLEN];
+ ulint buf_size;
+ xb_wf_incremental_ctxt_t *cp =
+ &(ctxt->u.wf_incremental_ctxt);
+
+ ctxt->cursor = cursor;
+
+ /* allocate buffer for incremental backup (4096 pages) */
+ buf_size = (cursor->page_size.physical() / 4 + 1)
+ * cursor->page_size.physical();
+ cp->delta_buf_base = static_cast(malloc(buf_size));
+ memset(cp->delta_buf_base, 0, buf_size);
+ cp->delta_buf = static_cast
+ (ut_align(cp->delta_buf_base, UNIV_PAGE_SIZE_MAX));
+
+ /* write delta meta info */
+ snprintf(meta_name, sizeof(meta_name), "%s%s", dst_name,
+ XB_DELTA_INFO_SUFFIX);
+ const xb_delta_info_t info(cursor->page_size, cursor->space_id);
+ if (!xb_write_delta_metadata(meta_name, &info)) {
+ msg("[%02u] xtrabackup: Error: "
+ "failed to write meta info for %s\n",
+ cursor->thread_n, cursor->rel_path);
+ return(FALSE);
+ }
+
+ /* change the target file name, since we are only going to write
+ delta pages */
+ strcat(dst_name, ".delta");
+
+ mach_write_to_4(cp->delta_buf, 0x78747261UL); /*"xtra"*/
+ cp->npages = 1;
+
+ return(TRUE);
+}
+
+/************************************************************************
+Run the next batch of pages through incremental page write filter.
+
+@return TRUE on success, FALSE on error. */
+static my_bool
+wf_incremental_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile)
+{
+ ulint i;
+ xb_fil_cur_t *cursor = ctxt->cursor;
+ byte *page;
+ const ulint page_size
+ = cursor->page_size.physical();
+ xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt);
+
+ for (i = 0, page = cursor->buf; i < cursor->buf_npages;
+ i++, page += page_size) {
+
+ if (incremental_lsn >= mach_read_from_8(page + FIL_PAGE_LSN)) {
+
+ continue;
+ }
+
+ /* updated page */
+ if (cp->npages == page_size / 4) {
+ /* flush buffer */
+ if (ds_write(dstfile, cp->delta_buf,
+ cp->npages * page_size)) {
+ return(FALSE);
+ }
+
+ /* clear buffer */
+ memset(cp->delta_buf, 0, page_size / 4 * page_size);
+ /*"xtra"*/
+ mach_write_to_4(cp->delta_buf, 0x78747261UL);
+ cp->npages = 1;
+ }
+
+ mach_write_to_4(cp->delta_buf + cp->npages * 4,
+ cursor->buf_page_no + i);
+ memcpy(cp->delta_buf + cp->npages * page_size, page,
+ page_size);
+
+ cp->npages++;
+ }
+
+ return(TRUE);
+}
+
+/************************************************************************
+Flush the incremental page write filter's buffer.
+
+@return TRUE on success, FALSE on error. */
+static my_bool
+wf_incremental_finalize(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile)
+{
+ xb_fil_cur_t *cursor = ctxt->cursor;
+ const ulint page_size
+ = cursor->page_size.physical();
+ xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt);
+
+ if (cp->npages != page_size / 4) {
+ mach_write_to_4(cp->delta_buf + cp->npages * 4, 0xFFFFFFFFUL);
+ }
+
+ /* Mark the final block */
+ mach_write_to_4(cp->delta_buf, 0x58545241UL); /*"XTRA"*/
+
+ /* flush buffer */
+ if (ds_write(dstfile, cp->delta_buf, cp->npages * page_size)) {
+ return(FALSE);
+ }
+
+ return(TRUE);
+}
+
+/************************************************************************
+Free the incremental page write filter's buffer. */
+static void
+wf_incremental_deinit(xb_write_filt_ctxt_t *ctxt)
+{
+ xb_wf_incremental_ctxt_t *cp = &(ctxt->u.wf_incremental_ctxt);
+
+ free(cp->delta_buf_base);
+}
+
+/************************************************************************
+Initialize the write-through page write filter.
+
+@return TRUE on success, FALSE on error. */
+static my_bool
+wf_wt_init(xb_write_filt_ctxt_t *ctxt, char *dst_name __attribute__((unused)),
+ xb_fil_cur_t *cursor)
+{
+ ctxt->cursor = cursor;
+
+ return(TRUE);
+}
+
+/************************************************************************
+Write the next batch of pages to the destination datasink.
+
+@return TRUE on success, FALSE on error. */
+static my_bool
+wf_wt_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile)
+{
+ xb_fil_cur_t *cursor = ctxt->cursor;
+
+ if (ds_write(dstfile, cursor->buf, cursor->buf_read)) {
+ return(FALSE);
+ }
+
+ return(TRUE);
+}
diff --git a/extra/mariabackup/write_filt.h b/extra/mariabackup/write_filt.h
new file mode 100644
index 00000000000..bcab263f1dd
--- /dev/null
+++ b/extra/mariabackup/write_filt.h
@@ -0,0 +1,58 @@
+/******************************************************
+XtraBackup: hot backup tool for InnoDB
+(c) 2009-2013 Percona LLC and/or its affiliates.
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+/* Page write filter interface */
+
+#ifndef XB_WRITE_FILT_H
+#define XB_WRITE_FILT_H
+
+#include "fil_cur.h"
+#include "datasink.h"
+
+/* Incremental page filter context */
+typedef struct {
+ byte *delta_buf_base;
+ byte *delta_buf;
+ ulint npages;
+} xb_wf_incremental_ctxt_t;
+
+/* Page filter context used as an opaque structure by callers */
+typedef struct {
+ xb_fil_cur_t *cursor;
+ union {
+ xb_wf_incremental_ctxt_t wf_incremental_ctxt;
+ } u;
+} xb_write_filt_ctxt_t;
+
+
+typedef struct {
+ my_bool (*init)(xb_write_filt_ctxt_t *ctxt, char *dst_name,
+ xb_fil_cur_t *cursor);
+ my_bool (*process)(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile);
+ my_bool (*finalize)(xb_write_filt_ctxt_t *, ds_file_t *dstfile);
+ void (*deinit)(xb_write_filt_ctxt_t *);
+} xb_write_filt_t;
+
+extern xb_write_filt_t wf_write_through;
+extern xb_write_filt_t wf_incremental;
+
+#endif /* XB_WRITE_FILT_H */
diff --git a/extra/mariabackup/wsrep.cc b/extra/mariabackup/wsrep.cc
new file mode 100644
index 00000000000..be11e058255
--- /dev/null
+++ b/extra/mariabackup/wsrep.cc
@@ -0,0 +1,220 @@
+/******************************************************
+Percona XtraBackup: hot backup tool for InnoDB
+(c) 2009-2014 Percona LLC and/or its affiliates
+Originally Created 3/3/2009 Yasufumi Kinoshita
+Written by Alexey Kopytov, Aleksandr Kuzminsky, Stewart Smith, Vadim Tkachenko,
+Yasufumi Kinoshita, Ignacio Nin and Baron Schwartz.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************
+
+This file incorporates work covered by the following copyright and
+permission notice:
+
+ Copyright 2010 Codership Oy
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; version 2 of the License.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#include
+#include
+#include
+#include
+
+#include "common.h"
+#ifdef WITH_WSREP
+#define WSREP_XID_PREFIX "WSREPXid"
+#define WSREP_XID_PREFIX_LEN MYSQL_XID_PREFIX_LEN
+#define WSREP_XID_UUID_OFFSET 8
+#define WSREP_XID_SEQNO_OFFSET (WSREP_XID_UUID_OFFSET + sizeof(wsrep_uuid_t))
+#define WSREP_XID_GTRID_LEN (WSREP_XID_SEQNO_OFFSET + sizeof(wsrep_seqno_t))
+
+/*! undefined seqno */
+#define WSREP_SEQNO_UNDEFINED (-1)
+
+/*! Name of file where Galera info is stored on recovery */
+#define XB_GALERA_INFO_FILENAME "xtrabackup_galera_info"
+
+/* Galera UUID type - for all unique IDs */
+typedef struct wsrep_uuid {
+ unsigned char data[16];
+} wsrep_uuid_t;
+
+/* sequence number of a writeset, etc. */
+typedef long long wsrep_seqno_t;
+
+/* Undefined UUID */
+static const wsrep_uuid_t WSREP_UUID_UNDEFINED = {{0,}};
+
+/***********************************************************************//**
+Check if a given WSREP XID is valid.
+
+@return true if valid.
+*/
+static
+bool
+wsrep_is_wsrep_xid(
+/*===============*/
+ const void* xid_ptr)
+{
+ const XID* xid = reinterpret_cast(xid_ptr);
+
+ return((xid->formatID == 1 &&
+ xid->gtrid_length == WSREP_XID_GTRID_LEN &&
+ xid->bqual_length == 0 &&
+ !memcmp(xid->data, WSREP_XID_PREFIX, WSREP_XID_PREFIX_LEN)));
+}
+
+/***********************************************************************//**
+Retrieve binary WSREP UUID from XID.
+
+@return binary WSREP UUID represenataion, if UUID is valid, or
+ WSREP_UUID_UNDEFINED otherwise.
+*/
+static
+const wsrep_uuid_t*
+wsrep_xid_uuid(
+/*===========*/
+ const XID* xid)
+{
+ if (wsrep_is_wsrep_xid(xid)) {
+ return(reinterpret_cast
+ (xid->data + WSREP_XID_UUID_OFFSET));
+ } else {
+ return(&WSREP_UUID_UNDEFINED);
+ }
+}
+
+/***********************************************************************//**
+Retrieve WSREP seqno from XID.
+
+@return WSREP seqno, if it is valid, or WSREP_SEQNO_UNDEFINED otherwise.
+*/
+wsrep_seqno_t wsrep_xid_seqno(
+/*==========================*/
+ const XID* xid)
+{
+ if (wsrep_is_wsrep_xid(xid)) {
+ wsrep_seqno_t seqno;
+ memcpy(&seqno, xid->data + WSREP_XID_SEQNO_OFFSET,
+ sizeof(wsrep_seqno_t));
+
+ return(seqno);
+ } else {
+ return(WSREP_SEQNO_UNDEFINED);
+ }
+}
+
+/***********************************************************************//**
+Write UUID to string.
+
+@return length of UUID string representation or -EMSGSIZE if string is too
+short.
+*/
+static
+int
+wsrep_uuid_print(
+/*=============*/
+ const wsrep_uuid_t* uuid,
+ char* str,
+ size_t str_len)
+{
+ if (str_len > 36) {
+ const unsigned char* u = uuid->data;
+ return snprintf(str, str_len,
+ "%02x%02x%02x%02x-%02x%02x-%02x%02x-"
+ "%02x%02x-%02x%02x%02x%02x%02x%02x",
+ u[ 0], u[ 1], u[ 2], u[ 3], u[ 4], u[ 5], u[ 6],
+ u[ 7], u[ 8], u[ 9], u[10], u[11], u[12], u[13],
+ u[14], u[15]);
+ }
+ else {
+ return -EMSGSIZE;
+ }
+}
+
+/***********************************************************************
+Store Galera checkpoint info in the 'xtrabackup_galera_info' file, if that
+information is present in the trx system header. Otherwise, do nothing. */
+void
+xb_write_galera_info(bool incremental_prepare)
+/*==================*/
+{
+ FILE* fp;
+ XID xid;
+ char uuid_str[40];
+ wsrep_seqno_t seqno;
+ MY_STAT statinfo;
+
+ /* Do not overwrite existing an existing file to be compatible with
+ servers with older server versions */
+ if (!incremental_prepare &&
+ my_stat(XB_GALERA_INFO_FILENAME, &statinfo, MYF(0)) != NULL) {
+
+ return;
+ }
+
+ memset(&xid, 0, sizeof(xid));
+ xid.formatID = -1;
+
+ if (!trx_sys_read_wsrep_checkpoint(&xid)) {
+
+ return;
+ }
+
+ if (wsrep_uuid_print(wsrep_xid_uuid(&xid), uuid_str,
+ sizeof(uuid_str)) < 0) {
+ return;
+ }
+
+ fp = fopen(XB_GALERA_INFO_FILENAME, "w");
+ if (fp == NULL) {
+
+ msg("xtrabackup: error: "
+ "could not create " XB_GALERA_INFO_FILENAME
+ ", errno = %d\n",
+ errno);
+ exit(EXIT_FAILURE);
+ }
+
+ seqno = wsrep_xid_seqno(&xid);
+
+ msg("xtrabackup: Recovered WSREP position: %s:%lld\n",
+ uuid_str, (long long) seqno);
+
+ if (fprintf(fp, "%s:%lld", uuid_str, (long long) seqno) < 0) {
+
+ msg("xtrabackup: error: "
+ "could not write to " XB_GALERA_INFO_FILENAME
+ ", errno = %d\n",
+ errno);
+ exit(EXIT_FAILURE);
+ }
+
+ fclose(fp);
+}
+#endif
diff --git a/extra/mariabackup/xb0xb.h b/extra/mariabackup/xb0xb.h
new file mode 100644
index 00000000000..59938a014c6
--- /dev/null
+++ b/extra/mariabackup/xb0xb.h
@@ -0,0 +1,27 @@
+/******************************************************
+Copyright (c) 2012 Percona LLC and/or its affiliates.
+
+Declarations of XtraBackup functions called by InnoDB code.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#ifndef xb0xb_h
+#define xb0xb_h
+
+extern const char *innodb_checksum_algorithm_names[];
+extern TYPELIB innodb_checksum_algorithm_typelib;
+
+#endif
diff --git a/extra/mariabackup/xb_regex.h b/extra/mariabackup/xb_regex.h
new file mode 100644
index 00000000000..2e07e434e27
--- /dev/null
+++ b/extra/mariabackup/xb_regex.h
@@ -0,0 +1,48 @@
+/******************************************************
+Copyright (c) 2011-2013 Percona LLC and/or its affiliates.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+/* This file is required to abstract away regex(3) calls so that
+my_regex is used on Windows and native calls are used on POSIX platforms. */
+
+#ifndef XB_REGEX_H
+#define XB_REGEX_H
+
+#ifdef HAVE_SYSTEM_REGEX
+#include
+#else
+#include
+#endif
+
+typedef regex_t* xb_regex_t;
+
+#define xb_regex_init()
+
+#define xb_regexec(preg,string,nmatch,pmatch,eflags) \
+ regexec(preg, string, nmatch, pmatch, eflags)
+
+#define xb_regerror(errcode,preg,errbuf,errbuf_size) \
+ regerror(errcode, preg, errbuf, errbuf_size)
+
+#define xb_regcomp(preg,regex,cflags) \
+ regcomp(preg, regex, cflags)
+
+#define xb_regfree(preg) regfree(preg)
+
+#define xb_regex_end()
+
+#endif /* XB_REGEX_H */
diff --git a/extra/mariabackup/xbcloud.cc b/extra/mariabackup/xbcloud.cc
new file mode 100644
index 00000000000..56661b03dd0
--- /dev/null
+++ b/extra/mariabackup/xbcloud.cc
@@ -0,0 +1,2721 @@
+/******************************************************
+Copyright (c) 2014 Percona LLC and/or its affiliates.
+
+The xbstream utility: serialize/deserialize files in the XBSTREAM format.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; version 2 of the License.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+*******************************************************/
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include