From 668f83c1635a29da5f12e865adde75dbe4f6fb32 Mon Sep 17 00:00:00 2001 From: Ben Thompson Date: Fri, 8 Feb 2019 15:15:37 -0600 Subject: [PATCH] end_to_end_test initial commit. --- utils/cloudio/CMakeLists.txt | 5 + utils/cloudio/end_to_end_test.cpp | 262 +++++++++++++++ .../testData/end_to_end_test_file.start | 308 ++++++++++++++++++ 3 files changed, 575 insertions(+) mode change 100644 => 100755 utils/cloudio/CMakeLists.txt create mode 100755 utils/cloudio/end_to_end_test.cpp create mode 100644 utils/cloudio/testData/end_to_end_test_file.start diff --git a/utils/cloudio/CMakeLists.txt b/utils/cloudio/CMakeLists.txt old mode 100644 new mode 100755 index 03375e68d..9e41ec45b --- a/utils/cloudio/CMakeLists.txt +++ b/utils/cloudio/CMakeLists.txt @@ -15,6 +15,8 @@ install(TARGETS cloudio DESTINATION ${ENGINE_LIBDIR} COMPONENT libs) add_executable(cloudio_component_test component_test.cpp) +add_executable(end_to_end_test end_to_end_test.cpp) + # see the comment above and change this dependency to cloudio. Hm # our lib dependencies seem not to be declared. Punting on that, # maybe in the future we can have some poor unfortunate intern @@ -22,5 +24,8 @@ add_executable(cloudio_component_test component_test.cpp) # For now I'm going to do like the other executables, which means # nearly everything AFAICT. target_link_libraries(cloudio_component_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS} cloudio) +target_link_libraries(end_to_end_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS} cloudio) #target_link_libraries(cloudio_component_test cloudio) +# Copy end_to_end_test dataFile to binary directory +FILE(COPY testData DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) diff --git a/utils/cloudio/end_to_end_test.cpp b/utils/cloudio/end_to_end_test.cpp new file mode 100755 index 000000000..5a063a077 --- /dev/null +++ b/utils/cloudio/end_to_end_test.cpp @@ -0,0 +1,262 @@ +/* Copyright (C) 2014 InfiniDB, Inc. + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + + + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "SMFileSystem.h" +#include "SMDataFile.h" +#include "SMFileFactory.h" +#include "sm_exceptions.h" +#include "messageFormat.h" + +/* The purpose of this is to test the idbdatafile cloud classes using a dummy +SessionManager defined in this file. */ + +#undef NDEBUG +#include + +using namespace idbdatafile; +using namespace std; + +volatile bool die = false; +int errCode = 0; + +void printResultPASS(int result) +{ + if (result == 0) + cout << " PASS" << endl; + else + cout << " FAIL" << endl; +} + +void printResultFAIL(int result) +{ + if (result == 0) + cout << " FAIL" << endl; + else + cout << " PASS" << endl; +} + +void printResultPASS(bool result) +{ + if (result) + cout << " PASS" << endl; + else + cout << " FAIL" << endl; +} + +void printResultFAIL(bool result) +{ + if (result) + cout << " FAIL" << endl; + else + cout << " PASS" << endl; +} + +void printResult(int result, int expected) +{ + if (result == expected) + cout << " PASS" << endl; + else + cout << " FAIL" << endl; +} + +int testAll() +{ + /* Instantiate each of SM subclasses, call each function, and verify the expected error response */ + int err; + SMFileFactory factory; + list filenames; + SMFileSystem filesystem; + + cout << "compressedSize --- "; + bool gotException = false; + try { + filesystem.compressedSize("./testData/dummy"); + } + catch (NotImplementedYet &) { + gotException = true; + } + assert(gotException && !die); + cout << " N/A" << endl; + + cout << "copyFile dummy1->dummy2 --- "; + err = filesystem.copyFile("./testData/dummy1", "./testData/dummy2"); + printResultFAIL(err); + + cout << "copyFile end_to_end_test_file.start->readTest --- "; + err = filesystem.copyFile("./testData/end_to_end_test_file.start", "./testData/readTest"); + printResultPASS(err); + + cout << "copyFile end_to_end_test_file.start->dummy --- "; + err = filesystem.copyFile("./testData/end_to_end_test_file.start", "./testData/dummy"); + printResultPASS(err); + + cout << "rename dummy->dummyrename --- "; + err = filesystem.rename("./testData/dummy", "./testData/dummyrename"); + printResultPASS(err); + + cout << "exists dummyrename --- "; + bool exists = filesystem.exists("./testData/dummyrename"); + printResultPASS(exists); + cout << "exists DNE --- "; + bool doesNotExist = filesystem.exists("./testData/DNE"); + printResultFAIL(doesNotExist); + + cout << "filesystemisup --- "; + bool fsIsUp = filesystem.filesystemIsUp(); + printResultPASS(fsIsUp); + + cout << "isdir testData --- "; + bool isDir = filesystem.isDir("testData"); + printResultPASS(isDir); + + cout << "isdir readTest --- "; + bool isNotDir = filesystem.isDir("./testData/readTest"); + printResultFAIL(isNotDir); + + cout << "isdir DNE --- "; + isNotDir = filesystem.isDir("./testData/DNE"); + printResultFAIL(isNotDir); + + cout << "listdirectory testData --- "; + err = filesystem.listDirectory("testData", filenames); + printResultPASS(err); + + cout << "listdirectory dummyrename --- "; + err = filesystem.listDirectory("./testData/dummyrename", filenames); + printResultFAIL(err); + + cout << "listdirectory DNE --- "; + err = filesystem.listDirectory("./testData/DNE", filenames); + printResultFAIL(err); + + cout << "remove dummyrename --- "; + err = filesystem.remove("./testData/dummyrename"); + printResultPASS(err); + + cout << "size readTest --- "; + err = filesystem.size("./testData/readTest"); + printResult(err,10940); + + cout << "size DNE --- "; + err = filesystem.size("./testData/DNE"); + printResultFAIL(err); + + cout << "open readTest r --- "; + IDBDataFile *fileR = factory.open("./testData/readTest", "r", 0, 0); + if (fileR) + cout << " PASS" << endl; + else + cout << " FAIL" << endl; + + cout << "open readTest a --- "; + IDBDataFile *fileW = factory.open("./testData/readTest", "a", 0, 0); + if (fileW) + cout << " PASS" << endl; + else + cout << " FAIL" << endl; + + cout << "open DNE --- "; + IDBDataFile *file2 = factory.open("./testData/DNE", "r", 0, 0); + if (file2) + cout << " FAIL" << endl; + else + cout << " PASS" << endl; + + cout << "pread --- "; + uint8_t buf[2048]; + err = fileR->pread(buf, 0, 1024); + printResult(err,1024); + + cout << "read --- "; + err = fileR->read(buf, 1024); + printResult(err,1024); + + int newSize = 10940 + 1024; + cout << "write --- "; + err = fileW->write(buf, 1024); + printResult(err,1024); + cout << "size "<< newSize << " --- "; + err = fileW->size(); + cout << err; + printResult(err,newSize); + + cout << "seek SEEK_SET --- "; + err = fileR->seek(1234, SEEK_SET); + printResultPASS(err); + cout << "tell 1234 --- "; + err = fileR->tell(); + cout << err; + printResult(err,1234); + cout << "seek SEEK_CUR --- "; + err = fileR->seek(1234, SEEK_CUR); + printResultPASS(err); + cout << "tell 2468 --- "; + err = fileR->tell(); + cout << err; + printResult(err,2468); + cout << "seek SEEK_END --- "; + err = fileR->seek(1234, SEEK_END); + printResultPASS(err); + cout << "tell 12174 --- "; + err = fileR->tell(); + cout << err; + printResult(err,13198); + + cout << "truncate 1234 --- "; + err = fileR->truncate(1234); + printResultPASS(err); + + cout << "size 1234 --- "; + err = fileR->size(); + printResult(err,1234); + + cout << "mtime --- "; + err = fileR->mtime(); + if (err > 0) + cout << " PASS" << endl; + else + cout << " FAIL" << endl; + + cout << "remove readTest --- "; + err = filesystem.remove("./testData/readTest"); + printResultPASS(err); + + cout << "done." << endl; + return errCode; +} + +int main() +{ + int ret; + + ret = testAll(); + //ret = testTest(); + return ret; +} + diff --git a/utils/cloudio/testData/end_to_end_test_file.start b/utils/cloudio/testData/end_to_end_test_file.start new file mode 100644 index 000000000..71b8d11f3 --- /dev/null +++ b/utils/cloudio/testData/end_to_end_test_file.start @@ -0,0 +1,308 @@ +# MariaDB ColumnStore Server (version 1.2) + +This is the server part of MariaDB ColumnStore 1.2. +MariaDB ColumnStore 1.2 is an GA of MariaDB ColumnStore. +It is built by porting InfiniDB 4.6.7 on MariaDB 10.3 and adding entirely +new features not found anywhere else. + +## MariaDB ColumnStore Engine (version 1.2) + +MariaDB ColumnStore also requires the matching engine version. This can be found at https://github.com/mariadb-corporation/mariadb-columnstore-engine. + +Always match the server engine / git branch with the engine git branch. + +Currently building has only been certified on CentOS 6 and 7, Ubuntu 16.04, Debain 8, and SUSE 12.. +Building on other platforms will be certified in a later release. + +## Issue tracking + +Issue tracking of MariaDB ColumnStore happens in JIRA, https://jira.mariadb.org/browse/MCOL + +### The structure of this repository is: + +* Branch "master" - this is the latest released version of the source code. Each major release is tagged. +* Branch "develop-1.0" - this is the 1.0 legacy mainline development branch. +* Branch "develop-1.1" - this is the 1.1 mainline development branch. +* Branch "develop" - this is the 1.2 unstable development branch. +* Branch "mcol-xxx" - these are specific bug and feature branches. These are merged into development which is merged to master. + +MariaDB ColumnStore server and the engine are in separate repositories. + +## Contributing + +To contribute to ColumnStore please see the [Contributions Documentation](CONTRIBUTING.md). + +## Build dependencies + +### Boost Libraries + +MariaDB ColumnStore requires that the boost package of 1.53 or newer is installed for both building and executing + +For CentOS 7, Ubuntu 16, Debian 8, SUSE 12 and other newer OS's, you can just install the boost packages via yum or apt-get. + +```bash +yum install boost-devel +``` + +or + +```bash +apt-get install libboost-dev-all +``` +or + +```bash +SUSEConnect -p sle-sdk/12.2/x86_64 + +zypper install boost-devel +``` + +For CentOS 6, you can either download and install the MariaDB Columnstore Centos 6 boost library package or install the boost source of 1.55 and build it to generate the required libraries. That means both the build and the install machines require this. + +Downloading and installing the MariaDB Columnstore Centos 6 boost library package is documented here: + +https://mariadb.com/kb/en/mariadb/preparing-for-columnstore-installation/#boost-libraries + +Downloading and build the boost libraries: + +NOTE: This means that the "Development Tools" group install be done prior to this. + +```bash +yum groupinstall "Development Tools" +yum install cmake +``` + +Here is the procedure to download and build the boost source: + +```bash +cd /usr/ + +wget http://sourceforge.net/projects/boost/files/boost/1.55.0/boost_1_55_0.tar.gz + +tar zxvf boost_1_55_0.tar.gz + +cd boost_1_55_0 + +./bootstrap.sh --with-libraries=atomic,date_time,exception,filesystem,iostreams,locale,program_options,regex,signals,system,test,thread,timer,log --prefix=/usr + +./b2 install + +ldconfig +``` + +### For CentOS + +These packages need to be install along with the group development packages: + +```bash +yum groupinstall "Development Tools" +yum install bison ncurses-devel readline-devel perl-devel openssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel python-devel ruby-devel tree wget pam-devel snappy-devel +``` + +### For Ubuntu 16/18 + +```bash +apt-get install build-essential automake libboost-all-dev bison cmake libncurses5-dev libreadline-dev libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev libsnappy-dev +``` +### For Debian 8 + +```bash +apt-get install build-essential automake libboost-all-dev bison cmake libncurses5-dev libreadline-dev libperl-dev libssl-dev libxml2-dev libkrb5-dev flex libpam-dev libkrb5-dev libsnappy-dev +``` + +### For Debian 9 + +```bash +apt-get install build-essential automake libboost-all-dev bison cmake libncurses5-dev libreadline-dev libperl-dev libssl1.0-dev libxml2-dev libkrb5-dev flex libpam-dev libkrb5-dev libsnappy-dev libgmp-dev +apt-get install libxml2-dev lmodern openjdk-8-jdk openssl pkg-config scons texlive-latex-base texlive-latex-base-doc thrift-compiler ttf-dejavu ttf-dejavu-extra unixodbc-dev uuid-dev xfonts-encodings xfonts-utils zlib1g-dev +``` +### For Suse 12 + +These packages need to be install along with the group development packages: + +```bash +zypper se -t pattern devel +zypper install bison ncurses-devel readline-devel perl-devel openssl-devel cmake libxml2-devel gperf libaio-devel libevent-devel python-devel ruby-devel tree wget pam-devel snappy-devel +``` + + +## Building master branch + +The current master branch is the released version. + +## Building develop branch + +The develop branch is used for develop updates + +Building can be done as a non-root user. If you do a "build install", it will install the binaries in `/usr/local/mariadb/columnstore` +and the use of sudo is required. + +To build the current development branch binaries only (Engine checkout inside Server): +```bash +git clone https://github.com/mariadb-corporation/mariadb-columnstore-server.git +cd mariadb-columnstore-server +git checkout develop # switch to develop code +cmake . -DCMAKE_INSTALL_PREFIX=/usr/local/mariadb/columnstore/mysql +make -jN # N is the number of concurrent build processes and should likely be the number of cores available +sudo make install +git clone https://github.com/mariadb-corporation/mariadb-columnstore-engine.git +cd mariadb-columnstore-engine +git checkout develop +cmake . +make -jN # same as above with respect to concurrent processes +sudo make install +``` + +To build the current development branch binaries and packages only (Engine checkout inside Server): +```bash +git clone https://github.com/mariadb-corporation/mariadb-columnstore-server.git +cd mariadb-columnstore-server +git checkout develop # switch to develop code +run cmake +For RPMs: +cmake . -DWITH_READLINE=1 -DRPM=centos6 -DPLUGIN_CONNECT=NO -DWITH_WSREP=OFF -DINSTALL_LAYOUT=RPM -DCMAKE_INSTALL_PREFIX=/usr/local/mariadb/columnstore/mysql -DCPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION=/usr/local +For DEBIANs: +cmake . -DWITH_READLINE=1 -DDEB=xenial -DPLUGIN_CONNECT=NO -DWITH_WSREP=OFF -DINSTALL_LAYOUT=DEB -DCMAKE_INSTALL_PREFIX=/usr/local/mariadb/columnstore/mysql/ -DCPACK_DEB_EXCLUDE_FROM_AUTO_FILELIST_ADDITION=/usr/local +make -jN # N is the number of concurrent build processes and should likely be the number of cores available +sudo make install +make package +git clone https://github.com/mariadb-corporation/mariadb-columnstore-engine.git +cd mariadb-columnstore-engine +git checkout develop +run cmake +For RPMs" +cmake . -DRPM=centos6 +For DEBIANs: +cmake . -DDEB=xenial +make -jN # same as above with respect to concurrent processes +sudo make install +make package +``` + +With the engine checked out in a separate location the following values need to be set by cmake command. + +```bash +SERVER_BUILD_INCLUDE_DIR=Path to the server build include directory. +SERVER_SOURCE_ROOT_DIR=Path the directory the server source checked out from github. +``` + +### Examples + +Engine not located inside server: + +```bash +git clone https://github.com/mariadb-corporation/mariadb-columnstore-server.git +git clone https://github.com/mariadb-corporation/mariadb-columnstore-engine.git +cd mariadb-columnstore-server +cmake . -DCMAKE_INSTALL_PREFIX=/usr/local/mariadb/columnstore/mysql +make -jN # N is the number of concurrent build processes and should likely be the number of cores available +sudo make install +cd ../mariadb-columnstore-engine +cmake . -DSERVER_BUILD_INCLUDE_DIR=../mariadb-columnstore-server/include -DSERVER_SOURCE_ROOT_DIR=../mariadb-columnstore-server +make -jN # same as above with respect to concurrent processes +sudo make install +``` + +Build out-of-source: + +```bash +git clone https://github.com/mariadb-corporation/mariadb-columnstore-server.git +git clone https://github.com/mariadb-corporation/mariadb-columnstore-engine.git +mkdir buildServer +cd buildServer +cmake ../mariadb-columnstore-server -DCMAKE_INSTALL_PREFIX=/usr/local/mariadb/columnstore/mysql +make -jN # N is the number of concurrent build processes and should likely be the number of cores available +sudo make install +cd .. +mkdir buildEngine +cd buildEngine +cmake ../mariadb-columnstore-engine -DSERVER_BUILD_INCLUDE_DIR=../buildServer/include -DSERVER_SOURCE_ROOT_DIR=../mariadb-columnstore-server +make -jN # same as above with respect to concurrent processes +sudo make install +``` + +To build a debug version + * Add `-DCMAKE_BUILD_TYPE=debug -DPLUGIN_TOKUDB=NO` to each of the cmake commands in the build process + * Do not mix release and debug versions of server and engine + +To develop a new branch/feature/pull request + * Fork the server repo from github `mariadb-corporation/mariadb-columnstore-server` + * Fork the engine report from github `mariadb-corporation/mariadb-columnstore-engine` + * `git checkout develop #branch in server` + * `git submodule update --init` + * `git branch new-branch-name` (this can be in engine or server code) + * `git checkout new-branch-name` + * Edit source files + * `git commit -m 'meaningful checkin comment'` + * `git push -u origin new-branch-name` + * Issue pull request for merge from new-branch-name into develop + * MariaDB ColumnStore team will evaluate the changes and may request further development or changes before merge + +## Run dependencies + +## For CentOS + +For CentOS 6 follow the install procedure for boost from the build Dependecy section above, with CentOS 7 you can just do: + +```bash +yum install boost +``` + +In addition these packages need to be install: + +```bash +yum install expect perl perl-DBI openssl zlib file sudo libaio rsync snappy net-tools nmap +``` + +## For Ubuntu 16/18 + +These packages need to be installed: + +```bash +apt-get install expect perl openssl file sudo libdbi-perl libboost-all-dev libreadline-dev rsync libsnappy1v5 net-tools nmap +``` + +## For Debian 8 + +These packages need to be installed: + +```bash +apt-get install expect perl openssl file sudo libdbi-perl libboost-all-dev libreadline-dev rsync libsnappy1 net-tools nmap +``` +## For Debian 9 + +These packages need to be installed: + +```bash +apt-get install expect perl openssl file sudo libdbi-perl libboost-all-dev libreadline-dev rsync net-tools libsnappy1v5 libreadline5 nmap +``` +## For SUSE 12 + +These packages need to be installed: + +SUSEConnect -p sle-sdk/12.2/x86_64 +zypper install boost-devel + +```bash +zypper install expect perl perl-DBI openssl file sudo libaio1 rsync net-tools libsnappy1 nmap +``` + +## MariaDB ColumnStore utilizes the System Logging for logging purposes + +So you will want to make sure that one of these system logging packages is installed: + + syslog, rsyslog, or syslog-ng + +## Configure and Starting of MariaDB ColumnStore + +Follow the binary package install instructions in the ColumnStore Getting Starter Guide: + + https://mariadb.com/kb/en/mariadb/columnstore-getting-started/ + +Commands to run as root user: + +```bash +cd /usr/local/mariadb/columnstore/bin/ +./post-install +./postConfigure +```