database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-08-07 03:22:57 +03:00
Code Activity

Merge branch 'stable-23.10' into json_depth-rucha

Browse Source
This commit is contained in:
Rucha Deodhar
2025-06-03 16:29:29 +05:30
committed by GitHub
parent 53870ac06f b2c0106c93
commit 313b4c2212
200 changed files with 2722 additions and 20812 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
utils/basic/string_utils.h Normal file
View File

@@ -0,0 +1,15 @@
#pragma once
#include <algorithm>
#include <string>
namespace datatypes
{
inline bool ASCIIStringCaseInsensetiveEquals(const std::string& left, const std::string& right)
{
auto asciiHigher = [](char c) { return (c >= 'a' && c <= 'z') ? c - 'a' + 'A' : c; };
return left.size() == right.size() &&
std::equal(left.begin(), left.end(), right.begin(),
[&asciiHigher](char l, char r) { return asciiHigher(l) == asciiHigher(r); });
}
} // namespace datatypes

10
utils/batchloader/CMakeLists.txt
View File

@@ -6,12 +6,4 @@ set(batchloader_LIB_SRCS batchloader.cpp)
columnstore_library(batchloader ${batchloader_LIB_SRCS})
add_dependencies(batchloader loggingcpp)
target_link_libraries(batchloader ${NETSNMP_LIBRARIES})
install(
TARGETS batchloader
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(batchloader ${NETSNMP_LIBRARIES} loggingcpp)

9
utils/cacheutils/CMakeLists.txt
View File

@@ -5,11 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(cacheutils_LIB_SRCS cacheutils.cpp)
columnstore_library(cacheutils ${cacheutils_LIB_SRCS})
add_dependencies(cacheutils loggingcpp)
install(
TARGETS cacheutils
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(cacheutils messageqcpp)

17
utils/cloudio/CMakeLists.txt
View File

@@ -14,22 +14,15 @@ columnstore_library(cloudio ${cloudio_LIB_SRCS})
# IDBDataFile currently depends on cloudio, which is backward. Once cloudio has been turned into a proper plugin for
# idbdatafile, we should be able to reverse the dependency like so:
target_link_libraries(cloudio idbdatafile messageqcpp loggingcpp)
install(
TARGETS cloudio
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(cloudio idbdatafile messageqcpp)
add_executable(cloudio_component_test component_test.cpp)
add_executable(end_to_end_test end_to_end_test.cpp)
add_executable(cloudio_end_to_end_test end_to_end_test.cpp)
# see the comment above and change this dependency to cloudio. Hm our lib dependencies seem not to be declared. Punting
# on that, maybe in the future we can have some poor unfortunate intern untangle all of that and declare lib
# dependencies properly. For now I'm going to do like the other executables, which means nearly everything AFAICT.
target_link_libraries(cloudio_component_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} cloudio)
target_link_libraries(end_to_end_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} cloudio)
# target_link_libraries(cloudio_component_test cloudio) Copy end_to_end_test dataFile to binary directory
columnstore_link(cloudio_component_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} cloudio)
columnstore_link(cloudio_end_to_end_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} cloudio)
# columnstore_link(cloudio_component_test cloudio) Copy end_to_end_test dataFile to binary directory
file(COPY testData DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)

11
utils/common/CMakeLists.txt
View File

@@ -15,13 +15,4 @@ set(common_LIB_SRCS
)
columnstore_library(common ${common_LIB_SRCS})
target_link_libraries(common boost_filesystem)
add_dependencies(common loggingcpp)
install(
TARGETS common
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(common boost_filesystem configcpp loggingcpp messageqcpp ${MARIADB_STRING_LIBS})

8
utils/common/collation.h
View File

@@ -275,12 +275,4 @@ class CollationAwareComparator : public Charset
}
};
inline bool ASCIIStringCaseInsensetiveEquals(const std::string& left, const std::string& right)
{
auto asciiHigher = [](char c) { return (c >= 'a' && c <= 'z') ? c - 'a' + 'A' : c; };
return left.size() == right.size() &&
std::equal(left.begin(), left.end(), right.begin(),
[&asciiHigher](char l, char r) { return asciiHigher(l) == asciiHigher(r); });
}
} // end of namespace datatypes

13
utils/compress/CMakeLists.txt
View File

@@ -11,19 +11,10 @@ set_property(DIRECTORY PROPERTY INCLUDE_DIRECTORIES "${dirs}")
set(compress_LIB_SRCS idbcompress.cpp)
add_definitions(-DNDEBUG)
columnstore_library(compress ${compress_LIB_SRCS})
add_dependencies(compress loggingcpp external_boost)
columnstore_link(compress PRIVATE loggingcpp boost_filesystem ${SNAPPY_LIBRARIES})
target_link_libraries(compress ${SNAPPY_LIBRARIES})
if(HAVE_LZ4)
message_once(STATUS "LINK WITH LZ4")
target_link_libraries(compress ${LZ4_LIBRARIES})
columnstore_link(compress PRIVATE ${LZ4_LIBRARIES})
endif()
install(
TARGETS compress
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)

16
utils/configcpp/CMakeLists.txt
View File

@@ -1,16 +1,4 @@
set(configcpp_LIB_SRCS configcpp.cpp xmlparser.cpp)
include_directories(${ENGINE_COMMON_INCLUDES})
# ########## next target ###############
set(configcpp_LIB_SRCS configcpp.cpp xmlparser.cpp configstream.cpp)
columnstore_library(configcpp ${configcpp_LIB_SRCS})
add_dependencies(configcpp loggingcpp)
target_compile_definitions(configcpp PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
install(
TARGETS configcpp
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(configcpp PRIVATE LibXml2::LibXml2 boost_filesystem)

12
utils/configcpp/Calpont.xml
View File

@@ -1,12 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<Calpont Version="V1.0.0">
<Message>
<Name>Message</Name>
<Dir>.</Dir>
<Length>875</Length>
<Version>V1.0</Version>
</Message>
<SystemConfig>
<SystemVersion>V1.0</SystemVersion>
</SystemConfig>
</Calpont>

12
utils/configcpp/XCalpont.xml
View File

@@ -1,12 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<XCalpont Version="V1.0.0">
<Message>
<Name>Message</Name>
<Dir>.</Dir>
<Length>875</Length>
<Version>V1.0</Version>
</Message>
<SystemConfig>
<SystemVersion>V1.0</SystemVersion>
</SystemConfig>
</XCalpont>

71
utils/configcpp/configcpp.cpp
View File

@@ -25,8 +25,6 @@
#include <stdexcept>
#include <iostream>
#include <sstream>
#include <fstream>
#include <vector>
#include <string>
using namespace std;
@@ -36,31 +34,20 @@ using namespace std;
using namespace boost;
namespace fs = boost::filesystem;
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <stdio.h>
#include <libxml/xmlmemory.h>
#include <libxml/parser.h>
#include <fcntl.h>
#include <errno.h>
#include <stdlib.h>
#ifdef HAVE_ALLOCA_H
#include <alloca.h>
#endif
#include <cstring>
//#define NDEBUG
#include <cassert>
#include <atomic>
#include <cstring>
#include <errno.h>
#include <fcntl.h>
#include <libxml/parser.h>
#include <libxml/xmlmemory.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include "configcpp.h"
#include "exceptclasses.h"
#include "installdir.h"
#include <tr1/unordered_map>
#include "bytestream.h"
namespace config
{
@@ -460,44 +447,6 @@ void Config::write(const string& configFile) const
}
}
void Config::writeConfigFile(messageqcpp::ByteStream msg) const
{
struct flock fl;
int fd;
// get config file name being udated
string fileName;
msg >> fileName;
fl.l_type = F_WRLCK; // write lock
fl.l_whence = SEEK_SET;
fl.l_start = 0;
fl.l_len = 0;
fl.l_pid = getpid();
// lock file if it exists
if ((fd = open(fileName.c_str(), O_WRONLY)) >= 0)
{
if (fcntl(fd, F_SETLKW, &fl) == -1)
throw runtime_error("Config::write: file lock error " + fileName);
ofstream out(fileName.c_str());
out << msg;
fl.l_type = F_UNLCK; // unlock
if (fcntl(fd, F_SETLK, &fl) == -1)
throw runtime_error("Config::write: file unlock error " + fileName);
close(fd);
}
else
{
ofstream out(fileName.c_str());
out << msg;
}
}
/* static */
void Config::deleteInstanceMap()
{

17
utils/configcpp/configcpp.h
View File

@@ -42,7 +42,6 @@ class ByteStream;
}
namespace config
{
/** @brief a config file I/F class
@@ -55,7 +54,6 @@ namespace config
class Config
{
public:
/** @brief Config factory method
*
* Creates a singleton Config object
@@ -96,8 +94,7 @@ class Config
* @param name the param name whose value is to be returned
* @param values the values in the section are returned in this vector
*/
void getConfig(const std::string& section, const std::string& name,
std::vector<std::string>& values);
void getConfig(const std::string& section, const std::string& name, std::vector<std::string>& values);
/** @brief set name's value in section
*
@@ -132,13 +129,6 @@ class Config
*/
void write(const std::string& fileName) const;
/** @brief write a stream copy of config file to disk
*
* write a stream copy of config file to disk. used to distributed mass updates to system nodes
*
*/
void writeConfigFile(messageqcpp::ByteStream msg) const;
/** @brief return the name of this config file
*
* return the name of this config file.
@@ -256,12 +246,10 @@ class Config
static Config& globConfigInstance();
Config(const Config& rhs);
Config& operator=(const Config& rhs);
Config(const std::string& configFile);
xmlDocPtr fDoc;
const std::string fConfigFile;
time_t fMtime;
@@ -272,11 +260,8 @@ class Config
*
*/
void checkAndReloadConfig();
};
} // namespace config
#undef EXPORT

66
utils/configcpp/configstream.cpp
View File

@@ -1,66 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/******************************************************************************************
* $Id$
*
******************************************************************************************/
#include "mcsconfig.h"
#include <string>
#include <libxml/xmlmemory.h>
#include <libxml/parser.h>
#include <stdexcept>
using namespace std;
#include "bytestream.h"
using namespace messageqcpp;
#include "configstream.h"
namespace config
{
ConfigStream::ConfigStream(const ByteStream& bs) : fParser()
{
init(reinterpret_cast<const xmlChar*>(bs.buf()));
}
ConfigStream::ConfigStream(const string& str) : fParser()
{
init(reinterpret_cast<const xmlChar*>(str.c_str()));
}
ConfigStream::ConfigStream(const char* cptr) : fParser()
{
init(reinterpret_cast<const xmlChar*>(cptr));
}
ConfigStream::~ConfigStream()
{
if (fDoc != NULL)
xmlFreeDoc(fDoc);
}
void ConfigStream::init(const xmlChar* xp)
{
fDoc = xmlParseDoc(xp);
if (fDoc == NULL)
throw runtime_error("ConfigStream::ConfigStream: bad XML stream");
}
} // namespace config

63
utils/configcpp/configstream.h
View File

@@ -1,63 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/******************************************************************************************
* $Id$
*
******************************************************************************************/
/**
* @file
*/
#pragma once
#include <string>
#include <libxml/parser.h>
#include "bytestream.h"
#include "xmlparser.h"
namespace config
{
/** @brief a config ByteStream I/F class
*
*/
class ConfigStream
{
public:
ConfigStream(const messageqcpp::ByteStream& bs);
ConfigStream(const std::string& str);
ConfigStream(const char* cptr);
~ConfigStream();
const std::string getConfig(const std::string& section, const std::string& name) const
{
return fParser.getConfig(fDoc, section, name);
}
private:
ConfigStream(const ConfigStream& rhs);
ConfigStream& operator=(const ConfigStream& rhs);
void init(const xmlChar* xp);
XMLParser fParser;
xmlDocPtr fDoc;
};
} // namespace config

14
utils/configcpp/resource.h
View File

@@ -1,14 +0,0 @@
//{{NO_DEPENDENCIES}}
// Microsoft Visual C++ generated include file.
// Used by libconfigcpp.rc
// Next default values for new objects
//
#ifdef APSTUDIO_INVOKED
#ifndef APSTUDIO_READONLY_SYMBOLS
#define _APS_NEXT_RESOURCE_VALUE 101
#define _APS_NEXT_COMMAND_VALUE 40001
#define _APS_NEXT_CONTROL_VALUE 1001
#define _APS_NEXT_SYMED_VALUE 101
#endif
#endif

371
utils/configcpp/tdriver.cpp
View File

@@ -1,371 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <string>
#include <stdexcept>
#include <sys/types.h>
#include <utime.h>
#include <sys/stat.h>
#include <iostream>
#include <iomanip>
#include <fstream>
using namespace std;
#include <boost/scoped_ptr.hpp>
using namespace boost;
#include <cppunit/extensions/HelperMacros.h>
#include "bytestream.h"
using namespace messageqcpp;
#include "configcpp.h"
#include "configstream.h"
using namespace config;
class ConfigFileTest : public CppUnit::TestFixture
{
CPPUNIT_TEST_SUITE(ConfigFileTest);
CPPUNIT_TEST(test1);
CPPUNIT_TEST_EXCEPTION(test2, std::runtime_error);
CPPUNIT_TEST(test3);
CPPUNIT_TEST(test4);
CPPUNIT_TEST_EXCEPTION(test5, std::runtime_error);
CPPUNIT_TEST_EXCEPTION(test6, std::runtime_error);
CPPUNIT_TEST_EXCEPTION(test7, std::invalid_argument);
CPPUNIT_TEST_EXCEPTION(test8, std::invalid_argument);
CPPUNIT_TEST(test9);
CPPUNIT_TEST(test10);
CPPUNIT_TEST(test11);
CPPUNIT_TEST(test12);
CPPUNIT_TEST_EXCEPTION(test13_1, std::runtime_error);
CPPUNIT_TEST_EXCEPTION(test13_2, std::runtime_error);
CPPUNIT_TEST(test14);
CPPUNIT_TEST_SUITE_END();
private:
public:
void setUp()
{
}
void tearDown()
{
}
void test1()
{
Config* c1 = Config::makeConfig("./Columnstore.xml");
string value;
value = c1->getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
value = c1->getConfig("Message", "xName");
CPPUNIT_ASSERT(value.size() == 0);
Config::deleteInstanceMap();
}
void test2()
{
Config* c1 = Config::makeConfig("./yadayada.xml");
string value;
value = c1->getConfig("Message", "Name");
CPPUNIT_ASSERT(value.size() == 0);
value = c1->getConfig("Message", "xName");
CPPUNIT_ASSERT(value.size() == 0);
Config::deleteInstanceMap();
}
void test3()
{
Config* c1;
string value;
for (int i = 0; i < 1000; i++)
{
c1 = Config::makeConfig("./Columnstore.xml");
value = c1->getConfig("Message", "Name");
assert(value == "Message");
}
Config::deleteInstanceMap();
}
void test4()
{
Config* c1 = Config::makeConfig("./Columnstore.xml");
string value;
value = c1->getConfig("SystemConfig", "SystemVersion");
c1->setConfig("SystemConfig", "SystemVersion", "2.2.versionversionversion");
value = c1->getConfig("SystemConfig", "SystemVersion");
CPPUNIT_ASSERT(value == "2.2.versionversionversion");
::unlink("./Columnstore.xml.new");
c1->write("./Columnstore.xml.new");
value = c1->getConfig("SystemConfig", "SystemVersion");
CPPUNIT_ASSERT(value == "2.2.versionversionversion");
c1->setConfig("SystemConfig", "SystemVersion1", "V1.x");
value = c1->getConfig("SystemConfig", "SystemVersion1");
CPPUNIT_ASSERT(value == "V1.x");
c1->setConfig("SystemConfig1", "SystemVersion1", "Vx.x");
value = c1->getConfig("SystemConfig1", "SystemVersion1");
CPPUNIT_ASSERT(value == "Vx.x");
c1->write("./Columnstore.xml.new");
Config* c2 = Config::makeConfig("./Columnstore.xml.new");
value = c2->getConfig("SystemConfig1", "SystemVersion1");
CPPUNIT_ASSERT(value == "Vx.x");
c2->setConfig("SystemConfig", "SystemVersion1", "V1.1");
value = c2->getConfig("SystemConfig", "SystemVersion1");
CPPUNIT_ASSERT(value == "V1.1");
c2->write();
Config::deleteInstanceMap();
}
void test5()
{
Config* c1 = Config::makeConfig("./Columnstore.xml");
c1->write("/cantwritethis");
Config::deleteInstanceMap();
}
void test6()
{
Config* c1 = Config::makeConfig("./XColumnstore.xml");
// compiler warning...we won't actually get here
c1 = 0;
}
void test7()
{
Config* c1 = Config::makeConfig("./Columnstore.xml");
string s;
string n;
string v;
c1->setConfig(s, n, v);
Config::deleteInstanceMap();
}
void test8()
{
Config* c1 = Config::makeConfig("./Columnstore.xml");
string s;
string n;
string v;
v = c1->getConfig(s, n);
Config::deleteInstanceMap();
}
void test9()
{
string value;
Config* c1 = Config::makeConfig("./Columnstore.xml");
Config* c2 = Config::makeConfig("./Columnstore.xml.new");
value = c1->getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
value = c2->getConfig("SystemConfig", "SystemVersion1");
CPPUNIT_ASSERT(value == "V1.1");
Config::deleteInstanceMap();
}
void test10()
{
string value;
setenv("CALPONT_CONFIG_FILE", "./Columnstore.xml", 1);
Config* c1 = Config::makeConfig();
value = c1->getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
Config::deleteInstanceMap();
}
void test11()
{
string value;
struct stat stat_buf;
struct utimbuf utime_buf;
CPPUNIT_ASSERT(stat("./Columnstore.xml.new", &stat_buf) == 0);
Config* c1 = Config::makeConfig("./Columnstore.xml.new");
value = c1->getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
utime_buf.actime = utime_buf.modtime = stat_buf.st_mtime + 1;
CPPUNIT_ASSERT(utime("./Columnstore.xml.new", &utime_buf) == 0);
value = c1->getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
Config::deleteInstanceMap();
}
void test12()
{
string value;
int64_t ival;
uint64_t uval;
value = "10";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == 10);
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == 10);
value = "0x10";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == 0x10);
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == 0x10);
value = "010";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == 010);
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == 010);
value = "-10";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == -10);
value = "10K";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024));
value = "10k";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024));
value = "10M";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024 * 1024));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024 * 1024));
value = "10m";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024 * 1024));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024 * 1024));
value = "10G";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024 * 1024 * 1024LL));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024 * 1024 * 1024ULL));
value = "10g";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024 * 1024 * 1024LL));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024 * 1024 * 1024ULL));
value = "10MB";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == (10 * 1024 * 1024));
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == (10 * 1024 * 1024));
value = "0x7afafafafafafafa";
ival = Config::fromText(value);
CPPUNIT_ASSERT(ival == 0x7afafafafafafafaLL);
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == 0x7afafafafafafafaULL);
value = "-0x7afafafafafafafa";
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == 0x8505050505050506ULL);
value = "-1";
uval = Config::uFromText(value);
CPPUNIT_ASSERT(uval == 0xffffffffffffffffULL);
}
void test13_1()
{
string value;
int64_t ival;
value = "2.2MB"; // invalid char causes throw
ival = Config::fromText(value);
}
void test13_2()
{
string value;
int64_t ival;
value = "10,000"; // invalid char causes throw
ival = Config::fromText(value);
}
void test14()
{
ByteStream bs;
ifstream ifs("./Columnstore.xml");
ifs >> bs;
string id(".");
string value;
{
ConfigStream cs(bs, id);
value = cs.getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
}
string bss(reinterpret_cast<const char*>(bs.buf()), bs.length());
{
ConfigStream cs(bss, id);
value = cs.getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
}
{
ConfigStream cs(bss.c_str(), id);
value = cs.getConfig("Message", "Name");
CPPUNIT_ASSERT(value == "Message");
}
}
};
CPPUNIT_TEST_SUITE_REGISTRATION(ConfigFileTest);
#include <cppunit/extensions/TestFactoryRegistry.h>
#include <cppunit/ui/text/TestRunner.h>
int main(int argc, char** argv)
{
CppUnit::TextUi::TestRunner runner;
CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry();
runner.addTest(registry.makeTest());
bool wasSuccessful = runner.run("", false);
return (wasSuccessful ? 0 : 1);
}

123
utils/configcpp/tdriver2.cpp
View File

@@ -1,123 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <string>
#include <stdexcept>
#include <sys/types.h>
#include <utime.h>
#include <sys/stat.h>
#include <iostream>
#include <iomanip>
using namespace std;
#include <boost/scoped_ptr.hpp>
using namespace boost;
#include <cppunit/extensions/HelperMacros.h>
#include "configcpp.h"
#include "writeonce.h"
using namespace config;
namespace
{
const string cf("./woparms.dat");
}
class WOConfigFileTest : public CppUnit::TestFixture
{
CPPUNIT_TEST_SUITE(WOConfigFileTest);
CPPUNIT_TEST(test1);
CPPUNIT_TEST_EXCEPTION(test2, std::runtime_error);
CPPUNIT_TEST_EXCEPTION(test3, std::runtime_error);
CPPUNIT_TEST_EXCEPTION(test4, std::runtime_error);
CPPUNIT_TEST(test5);
CPPUNIT_TEST_SUITE_END();
private:
public:
void setUp()
{
unlink(cf.c_str());
}
void tearDown()
{
unlink(cf.c_str());
}
void test1()
{
WriteOnceConfig woc(cf);
CPPUNIT_ASSERT(woc.owns("SystemConfig", "DBRootCount"));
CPPUNIT_ASSERT(woc.owns("SystemConfig", "DBRMRoot"));
CPPUNIT_ASSERT(!woc.owns("dummy", "dummy"));
int vali;
woc.setConfig("SystemConfig", "DBRootCount", "10");
vali = Config::fromText(woc.getConfig("SystemConfig", "DBRootCount"));
CPPUNIT_ASSERT(vali == 10);
WriteOnceConfig woc2(cf.c_str());
vali = Config::fromText(woc2.getConfig("SystemConfig", "DBRootCount"));
CPPUNIT_ASSERT(vali == 10);
}
void test2()
{
WriteOnceConfig woc(cf);
woc.getConfig("dummy", "dummy");
}
void test3()
{
WriteOnceConfig woc(cf);
woc.setConfig("dummy", "dummy", "100");
}
void test4()
{
WriteOnceConfig woc(cf);
woc.setConfig("SystemConfig", "DBRootCount", "10");
woc.setConfig("SystemConfig", "DBRootCount", "11");
}
void test5()
{
WriteOnceConfig woc(cf);
woc.setConfig("SystemConfig", "DBRootCount", "10");
woc.setConfig("SystemConfig", "DBRootCount", "11", true);
}
};
CPPUNIT_TEST_SUITE_REGISTRATION(WOConfigFileTest);
#include <cppunit/extensions/TestFactoryRegistry.h>
#include <cppunit/ui/text/TestRunner.h>
int main(int argc, char** argv)
{
CppUnit::TextUi::TestRunner runner;
CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry();
runner.addTest(registry.makeTest());
bool wasSuccessful = runner.run("", false);
return (wasSuccessful ? 0 : 1);
}

5
utils/configcpp/xmlparser.cpp
View File

@@ -19,7 +19,9 @@
* $Id$
*
******************************************************************************************/
#include "mcsconfig.h"
// #include "mcsconfig.h"
#include "basic/string_utils.h"
#include <string>
#include <stdexcept>
@@ -27,7 +29,6 @@
#include <libxml/parser.h>
#include <vector>
#include "collation.h"
using namespace std;
#include "xmlparser.h"

9
utils/dataconvert/CMakeLists.txt
View File

@@ -5,11 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(dataconvert_LIB_SRCS dataconvert.cpp)
columnstore_library(dataconvert ${dataconvert_LIB_SRCS})
add_dependencies(dataconvert loggingcpp)
install(
TARGETS dataconvert
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(dataconvert PUBLIC loggingcpp datatypes)

2
utils/dataconvert/dataconvert.cpp
View File

@@ -38,6 +38,8 @@ using namespace boost::algorithm;
#include "calpontselectexecutionplan.h"
#include "columnresult.h"
#include "basic/string_utils.h"
#include "joblisttypes.h"
#define DATACONVERT_DLLEXPORT

1375
utils/dataconvert/tdriver.cpp
View File

File diff suppressed because it is too large Load Diff

14
utils/ddlcleanup/CMakeLists.txt
View File

@@ -1,17 +1,5 @@
include_directories(${ENGINE_COMMON_INCLUDES})
# ########## next target ###############
set(ddlcleanuputil_LIB_SRCS ddlcleanuputil.cpp)
columnstore_library(ddlcleanuputil ${ddlcleanuputil_LIB_SRCS})
add_dependencies(ddlcleanuputil loggingcpp)
target_link_libraries(ddlcleanuputil ${NETSNMP_LIBRARIES})
install(
TARGETS ddlcleanuputil
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(ddlcleanuputil PRIVATE loggingcpp ${NETSNMP_LIBRARIES})

18
utils/funcexp/CMakeLists.txt
View File

@@ -143,13 +143,13 @@ set(funcexp_LIB_SRCS
)
columnstore_library(funcexp ${funcexp_LIB_SRCS})
add_dependencies(funcexp loggingcpp)
target_link_libraries(funcexp ${NETSNMP_LIBRARIES} pron pcre2-8 ${MARIADB_STRING_LIBS})
install(
TARGETS funcexp
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
columnstore_link(
funcexp
PRIVATE
pcre2-8
pron
loggingcpp
dataconvert
${MARIADB_STRING_LIBS}
${NETSNMP_LIBRARIES}
)

7
utils/funcexp/func_json_contains_path.cpp
View File

@@ -34,7 +34,7 @@ bool Func_json_contains_path::getBoolVal(Row& row, FunctionParm& fp, bool& isNul
const string_view js = js_ns.unsafeStringRef();
#ifdef MYSQL_GE_1009
#if MYSQL_VERSION_ID >= 100900
int arrayCounters[JSON_DEPTH_LIMIT];
bool hasNegPath = false;
#endif
@@ -81,7 +81,7 @@ bool Func_json_contains_path::getBoolVal(Row& row, FunctionParm& fp, bool& isNul
isNull = true;
return false;
}
#ifdef MYSQL_GE_1009
#if MYSQL_VERSION_ID >= 100900
hasNegPath |= path.p.types_used & JSON_PATH_NEGATIVE_INDEX;
#endif
}
@@ -113,7 +113,6 @@ bool Func_json_contains_path::getBoolVal(Row& row, FunctionParm& fp, bool& isNul
while (json_get_path_next(&jsEg, &p) == 0)
{
#ifdef MYSQL_GE_1009
json_path_step_t *last_step= (json_path_step_t*)
(mem_root_dynamic_array_get_val(&p.steps,
p.last_step_idx));
@@ -128,7 +127,7 @@ bool Func_json_contains_path::getBoolVal(Row& row, FunctionParm& fp, bool& isNul
for (int restSize = argSize, curr = 0; restSize > 0; restSize--, curr++)
{
JSONPath& path = paths[curr];
#ifdef MYSQL_GE_1009
#if MYSQL_VERSION_ID >= 100900
int cmp = cmpJSPath(&path.p, &p, jsEg.value_type, arrayCounters);
#else
int cmp = cmpJSPath(&path.p, &p, jsEg.value_type);

9
utils/funcexp/func_json_extract.cpp
View File

@@ -31,7 +31,7 @@ int Func_json_extract::doExtract(Row& row, FunctionParm& fp, json_value_types* t
bool mayMulVal;
int wildcards;
bool isMatch;
#ifdef MYSQL_GE_1009
#if MYSQL_VERSION_ID >= 100900
int arrayCounter[JSON_DEPTH_LIMIT];
bool hasNegPath = false;
#endif
@@ -62,12 +62,12 @@ int Func_json_extract::doExtract(Row& row, FunctionParm& fp, json_value_types* t
if (!path.parsed && parseJSPath(path, row, fp[i]))
return 1;
#ifdef MYSQL_GE_1009
#if MYSQL_VERSION_ID >= 100900
hasNegPath |= path.p.types_used & JSON_PATH_NEGATIVE_INDEX;
#endif
}
#ifdef MYSQL_GE_1009
#if MYSQL_VERSION_ID >= 100900
wildcards = (JSON_PATH_WILD | JSON_PATH_DOUBLE_WILD | JSON_PATH_ARRAY_RANGE);
#else
wildcards = (JSON_PATH_WILD | JSON_PATH_DOUBLE_WILD);
@@ -87,14 +87,13 @@ int Func_json_extract::doExtract(Row& row, FunctionParm& fp, json_value_types* t
while (json_get_path_next(&jsEg, &p) == 0)
{
#ifdef MYSQL_GE_1009
json_path_step_t *last_step= (json_path_step_t*)(mem_root_dynamic_array_get_val(&p.steps, p.last_step_idx));
if (hasNegPath && jsEg.value_type == JSON_VALUE_ARRAY &&
json_skip_array_and_count(&jsEg, arrayCounter + (last_step - (json_path_step_t*)(p.steps.buffer))))
return 1;
#endif
#ifdef MYSQL_GE_1009
#if MYSQL_VERSION_ID >= 100900
isMatch = matchJSPath(paths, &p, jsEg.value_type, arrayCounter, false);
#else
isMatch = matchJSPath(paths, &p, jsEg.value_type, nullptr, false);

8
utils/funcexp/func_json_search.cpp
View File

@@ -141,7 +141,7 @@ string Func_json_search::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& i
json_path_step_t savPath_steps[JSON_DEPTH_LIMIT], p_steps[JSON_DEPTH_LIMIT];
const CHARSET_INFO* cs = getCharset(fp[0]);
#ifdef MYSQL_GE_1009
#if MYSQL_VERSION_ID >= 100900
int arrayCounter[JSON_DEPTH_LIMIT];
bool hasNegPath = 0;
#endif
@@ -167,7 +167,7 @@ string Func_json_search::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& i
JSON_DEPTH_LIMIT, 0, MYF(0));
if (parseJSPath(path, row, fp[i]))
goto error;
#ifdef MYSQL_GE_1009
#if MYSQL_VERSION_ID >= 100900
hasNegPath |= path.p.types_used & JSON_PATH_NEGATIVE_INDEX;
#endif
}
@@ -181,7 +181,7 @@ string Func_json_search::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& i
while (json_get_path_next(&jsEg, &p) == 0)
{
#ifdef MYSQL_GE_1009
#if MYSQL_VERSION_ID >= 100900
if (hasNegPath && jsEg.value_type == JSON_VALUE_ARRAY &&
json_skip_array_and_count(&jsEg, arrayCounter + (((json_path_step_t*)(mem_root_dynamic_array_get_val(&p.steps, p.last_step_idx))) - (json_path_step_t*)p.steps.buffer)))
goto error;
@@ -189,7 +189,7 @@ string Func_json_search::getStrVal(rowgroup::Row& row, FunctionParm& fp, bool& i
if (json_value_scalar(&jsEg))
{
#ifdef MYSQL_GE_1009
#if MYSQL_VERSION_ID >= 100900
bool isMatch = matchJSPath(paths, &p, jsEg.value_type, arrayCounter);
#else
bool isMatch = matchJSPath(paths, &p, jsEg.value_type);

4
utils/funcexp/funcexp.cpp
View File

@@ -163,7 +163,7 @@ FuncExp::FuncExp()
fFuncMap["json_merge_preserve"] = new Func_json_merge();
fFuncMap["json_normalize"] = new Func_json_normalize();
fFuncMap["json_object"] = new Func_json_object();
#ifdef MYSQL_GE_1009
#if MYSQL_VERSION_ID >= 100900
fFuncMap["json_overlaps"] = new Func_json_overlaps();
#endif
fFuncMap["json_query"] = new Func_json_query();
@@ -218,7 +218,7 @@ FuncExp::FuncExp()
fFuncMap["regexp"] = new Func_regexp();
fFuncMap["regexp_instr"] = new Func_regexp_instr();
fFuncMap["regexp_replace"] = new Func_regexp_replace();
fFuncMap["regexp_substr"] = new Func_regexp_substr(); // dlh
fFuncMap["regexp_substr"] = new Func_regexp_substr(); // dlh
fFuncMap["repeat"] = new Func_repeat(); // dlh
fFuncMap["replace"] = new Func_replace(); // dlh
fFuncMap["replace_oracle"] = new Func_replace_oracle(); // dlh

7
utils/funcexp/functor_json.h
View File

@@ -12,13 +12,6 @@
#include "functor_int.h"
#include "functor_str.h"
// Check if mariadb version >= 10.9
#if MYSQL_VERSION_ID >= 100900
#ifndef MYSQL_GE_1009
#define MYSQL_GE_1009
#endif
#endif
namespace funcexp
{
// The json_path_t wrapper include some flags

6
utils/funcexp/jsonhelpers.cpp
View File

@@ -13,7 +13,7 @@ int setupJSPath(json_path_t* path, CHARSET_INFO* cs, const utils::NullString& st
if (!err)
{
#ifdef MYSQL_GE_1009
#if MYSQL_VERSION_ID >= 100900
bool support = (path->types_used & (JSON_PATH_WILD | JSON_PATH_DOUBLE_WILD | JSON_PATH_ARRAY_RANGE)) == 0;
#else
bool support = (path->types_used & (JSON_PATH_WILD | JSON_PATH_DOUBLE_WILD)) == 0;
@@ -252,7 +252,7 @@ int cmpPartJSPath(const json_path_step_t* a, const json_path_step_t* aEnd, const
{
if (b->type & JSON_PATH_ARRAY)
{
#ifdef MYSQL_GE_1009
#if MYSQL_VERSION_ID >= 100900
int ret = 0, corrected_n_item_a = 0;
if (arraySize)
corrected_n_item_a = a->n_item < 0 ? arraySize[b - tmpB] + a->n_item : a->n_item;
@@ -360,7 +360,7 @@ bool matchJSPath(const vector<funcexp::JSONPath>& paths, const json_path_t* p, j
{
for (size_t curr = 0; curr < paths.size(); curr++)
{
#ifdef MYSQL_GE_1009
#if MYSQL_VERSION_ID >= 100900
int cmp = cmpJSPath(&paths[curr].p, p, valType, arrayCounter);
#else
int cmp = cmpJSPath(&paths[curr].p, p, valType);

2
utils/funcexp/jsonhelpers.h
View File

@@ -49,7 +49,7 @@ static const int TRIVIAL_PATH_NOT_ALLOWED = 3;
bool findKeyInObject(json_engine_t* jsEg, json_string_t* key);
#ifdef MYSQL_GE_1009
#if MYSQL_VERSION_ID >= 100900
using IntType = int;
#else
using IntType = uint;

11
utils/idbdatafile/CMakeLists.txt
View File

@@ -14,13 +14,4 @@ set(idbdatafile_LIB_SRCS
)
columnstore_library(idbdatafile ${idbdatafile_LIB_SRCS})
target_link_libraries(idbdatafile ${NETSNMP_LIBRARIES} ${ENGINE_OAM_LIBS} boost_filesystem boost_system)
target_compile_definitions(idbdatafile PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
install(
TARGETS idbdatafile
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(idbdatafile PRIVATE ${NETSNMP_LIBRARIES} ${ENGINE_OAM_LIBS} boost_filesystem boost_system)

9
utils/joiner/CMakeLists.txt
View File

@@ -5,11 +5,4 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(joiner_LIB_SRCS tuplejoiner.cpp joinpartition.cpp)
columnstore_library(joiner ${joiner_LIB_SRCS})
add_dependencies(joiner loggingcpp)
install(
TARGETS joiner
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(joiner PRIVATE loggingcpp)

6
utils/libmarias3/CMakeLists.txt
View File

@@ -15,10 +15,10 @@ set(S3_SOURCES
${S3API_DIR}/src/assume_role.c
)
add_library(marias3 SHARED ${S3_SOURCES})
target_link_libraries(marias3 curl m)
columnstore_shared_library(marias3 ${S3_SOURCES})
columnstore_link(marias3 curl m)
include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${S3API_DIR})
add_definitions(-D_GNU_SOURCE)
set(S3API_DEPS

13
utils/libmysql_client/CMakeLists.txt
View File

@@ -3,17 +3,6 @@ add_definitions(-DMYSQL_SERVICE_THD_TIMEZONE_INCLUDED)
include_directories(${ENGINE_COMMON_INCLUDES})
# ########## next target ###############
set(libmysql_client_LIB_SRCS libmysql_client.cpp)
columnstore_library(libmysql_client ${libmysql_client_LIB_SRCS})
target_link_libraries(libmysql_client ${MARIADB_CLIENT_LIBS})
add_dependencies(libmysql_client loggingcpp)
install(
TARGETS libmysql_client
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(libmysql_client PRIVATE loggingcpp ${MARIADB_CLIENT_LIBS})

8
utils/libmysql_client/libmysql_client.cpp
View File

@@ -96,6 +96,14 @@ int LibMySQL::init(const char* h, unsigned int p, const char* u, const char* w,
return ret;
}
// Disable Select Handler to avoid recursive SH execution path for CES.
static const std::string disableSelectHandler = "SET SESSION columnstore_select_handler = OFF";
if (mysql_real_query(fCon, disableSelectHandler.c_str(), disableSelectHandler.length()) != 0)
{
fErrStr = "fatal error setting columnstore_select_handler=OFF in libmysql_client lib";
ret = -1;
return ret;
}
return ret;
}

14
utils/loggingcpp/CMakeLists.txt
View File

@@ -31,16 +31,8 @@ columnstore_library(
${CMAKE_CURRENT_BINARY_DIR}/messageids.h
${CMAKE_CURRENT_BINARY_DIR}/errorids.h
)
add_dependencies(loggingcpp external_boost)
install(
TARGETS loggingcpp
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(loggingcpp configcpp boost_filesystem)
install(
FILES MessageFile.txt ErrorMessage.txt
DESTINATION ${ENGINE_SYSCONFDIR}/columnstore
COMPONENT columnstore-engine
)
columnstore_install_file(MessageFile.txt ${ENGINE_SYSCONFDIR}/columnstore)
columnstore_install_file(ErrorMessage.txt ${ENGINE_SYSCONFDIR}/columnstore)

7
utils/loggingcpp/message.cpp
View File

@@ -29,7 +29,6 @@
#include <fstream>
using namespace std;
#include <boost/tokenizer.hpp>
#include <boost/thread.hpp>
using namespace boost;
@@ -42,7 +41,6 @@ using namespace config;
#include "installdir.h"
#include "format.h"
#include "mcs_int128.h"
namespace
{
@@ -129,11 +127,6 @@ void Message::Args::add(uint64_t u64)
fArgs.push_back(u64);
}
void Message::Args::add(int128_t i128)
{
fArgs.push_back(datatypes::TSInt128(i128).toString());
}
void Message::Args::add(const string& s)
{
fArgs.push_back(s);

21
utils/messageqcpp/CMakeLists.txt
View File

@@ -1,21 +1,16 @@
include_directories(${ENGINE_COMMON_INCLUDES})
# ########## next target ###############
set(messageqcpp_LIB_SRCS
bytestream.cpp
bytestreampool.cpp
compressed_iss.cpp
inetstreamsocket.cpp
iosocket.cpp
messagequeue.cpp
messagequeuepool.cpp
bytestream.cpp
socketparms.cpp
inetstreamsocket.cpp
samenodepseudosocket.cpp
iosocket.cpp
compressed_iss.cpp
bytestreampool.cpp
socketparms.cpp
)
add_library(messageqcpp STATIC ${messageqcpp_LIB_SRCS})
add_dependencies(messageqcpp loggingcpp)
# We don't isntall static library install(TARGETS messageqcpp DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine)
columnstore_static_library(messageqcpp ${messageqcpp_LIB_SRCS})
columnstore_link(messageqcpp PUBLIC loggingcpp compress)

33
utils/multicast/CMakeLists.txt
View File

@@ -1,33 +0,0 @@
#
# Not used
#
# original Makefile.am contents follow:
# Copyright (C) 2014 InfiniDB, Inc.
#
# This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with this program; if not, write to the Free
# Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# $Id$ Process this file with automake to produce Makefile.in
#
# AM_CPPFLAGS = $(idb_common_includes) $(idb_cppflags) AM_CFLAGS = $(idb_cflags) AM_CXXFLAGS = $(idb_cxxflags)
# AM_LDFLAGS = -version-info 1:0:0 $(idb_ldflags) lib_LTLIBRARIES = libmulticast.la libmulticast_la_SOURCES =
# multicast.cpp include_HEADERS = multicast.h
#
# test:
#
# coverage:
#
# leakcheck:
#
# docs:
#
# bootstrap: install-data-am
#

3820
utils/multicast/impl.cpp
View File

File diff suppressed because it is too large Load Diff

63
utils/multicast/impl.h
View File

@@ -1,63 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/******************************************************************************
* $Id: impl.h 3495 2013-01-21 14:09:51Z rdempsey $
*
*****************************************************************************/
/* This code is based on udpcast-20090830. Most of the source code in that release
contains no copyright or licensing notices at all. The exception is fec.c, which
is not used here. The udpcast website, http://udpcast.linux.lu/, implies that
the source is covered under GPL. */
/** @file */
#pragma once
#include <string>
#include <stdint.h>
#include "bytestream.h"
#include "udpcast.h"
#include "participants.h"
namespace multicast
{
class MulticastImpl
{
public:
MulticastImpl(int min_receivers, const std::string& ifName, int portBase = 9000,
int bufSize = 8 * 1024 * 1024);
~MulticastImpl();
void startSender();
void doTransfer(const uint8_t* buf, uint32_t len);
void startReceiver();
void receive(messageqcpp::SBS obs);
struct net_config fNet_config;
struct stat_config fStat_config;
struct client_config fClient_config;
std::string fIfName;
int fSock[3];
participantsDb_t fDb;
};
} // namespace multicast

33
utils/multicast/mc_fifo.h
View File

@@ -1,33 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "threads.h"
#include "produconsum.h"
typedef struct fifo
{
unsigned char* dataBuffer;
unsigned int dataBufSize;
produconsum_t freeMemQueue; /* queue for free memory */
produconsum_t data; /* queue for received data or data received
* from disk */
pthread_t thread;
} * fifo_t;

95
utils/multicast/multicast.cpp
View File

@@ -1,95 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/******************************************************************************
* $Id: multicast.cpp 3495 2013-01-21 14:09:51Z rdempsey $
*
*****************************************************************************/
#include <string>
#include <stdint.h>
#include <iostream>
#include <stdexcept>
using namespace std;
#include "bytestream.h"
using namespace messageqcpp;
#include "configcpp.h"
using namespace config;
#include "multicast.h"
namespace multicast
{
Multicast::Multicast() : fPMCount(1), fIFName("eth0"), fPortBase(9000), fBufSize(8 * 1024 * 1024)
{
int tmp;
string stmp;
Config* cf = Config::makeConfig();
tmp = Config::fromText(cf->getConfig("PrimitiveServers", "Count"));
if (tmp > 0)
fPMCount = tmp;
stmp = cf->getConfig("Multicast", "Interface");
if (!stmp.empty())
fIFName = stmp;
tmp = Config::fromText(cf->getConfig("Multicast", "PortBase"));
if (tmp > 0)
fPortBase = tmp;
tmp = Config::fromText(cf->getConfig("Multicast", "BufSize"));
if (tmp > 0)
fBufSize = tmp;
}
MulticastReceiver::MulticastReceiver() : fPimpl(0)
{
}
MulticastReceiver::~MulticastReceiver()
{
}
SBS MulticastReceiver::receive()
{
throw runtime_error("Multicast is not available");
return fByteStream;
}
MulticastSender::MulticastSender() : fPimpl(0)
{
}
MulticastSender::~MulticastSender()
{
}
void MulticastSender::send(const ByteStream& msg)
{
throw runtime_error("Multicast is not available");
}
} // namespace multicast

128
utils/multicast/multicast.h
View File

@@ -1,128 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/******************************************************************************
* $Id: multicast.h 3495 2013-01-21 14:09:51Z rdempsey $
*
*****************************************************************************/
/** @file
* class Multicast interface
*/
#pragma once
#include "messagequeue.h"
namespace multicast
{
/** @brief MulticastReceive
* Wrapper for multicast proto
*/
class MulticastImpl;
class Multicast
{
public:
/** @brief ctor
* Base class
*/
Multicast();
/** @brief dtor
*/
virtual ~Multicast()
{
destroy();
}
virtual void destroy()
{
}
int PMCount() const
{
return fPMCount;
}
std::string iFName() const
{
return fIFName;
}
int portBase() const
{
return fPortBase;
}
int bufSize() const
{
return fBufSize;
}
private:
int fPMCount;
std::string fIFName;
int fPortBase;
int fBufSize;
};
class MulticastReceiver : public Multicast
{
public:
/** @brief ctor
*
*/
MulticastReceiver();
~MulticastReceiver();
messageqcpp::SBS receive();
private:
// not copyable
MulticastReceiver(const MulticastReceiver& rhs);
MulticastReceiver& operator=(const MulticastReceiver& rhs);
messageqcpp::SBS fByteStream;
MulticastImpl* fPimpl;
};
class MulticastSender : public Multicast
{
public:
/** @brief ctor
*
*/
MulticastSender();
~MulticastSender();
/** @brief receive
*
* @param bytestream to send
*/
void send(const messageqcpp::ByteStream& bs);
private:
// Not copyable
MulticastSender(const MulticastSender& rhs);
MulticastSender& operator=(const MulticastSender& rhs);
MulticastImpl* fPimpl;
};
} // namespace multicast

28
utils/multicast/participants.h
View File

@@ -1,28 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#define MAX_CLIENTS 1024
typedef struct participantsDb* participantsDb_t;
#define printNotSet udpc_printNotSet
#define printSet udpc_printSet
void udpc_printNotSet(participantsDb_t db, char* d);
void udpc_printSet(participantsDb_t db, char* d);

25
utils/multicast/produconsum.h
View File

@@ -1,25 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
typedef struct produconsum* produconsum_t;
/**
* Get contiguous chunk of data
*/
int pc_consumeAnyContiguous(produconsum_t pc);

169
utils/multicast/socklib.h
View File

@@ -1,169 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#ifndef UDPCAST_CONFIG_H
#define UDPCAST_CONFIG_H
#include "mcsconfig.h"
#endif
#include <sys/types.h>
#include <string.h>
#ifdef HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif
#ifdef HAVE_SYS_SELECT_H
#include <sys/select.h>
#endif
#ifdef HAVE_SYS_SOCKIO_H
#include <sys/sockio.h>
#endif
#ifdef HAVE_NETINET_IN_H
#include <netinet/in.h>
#endif
#ifdef HAVE_NETDB_H
#include <netdb.h>
#endif
#ifdef HAVE_WINSOCK2_H
#include <winsock2.h>
#endif
#ifdef HAVE_SYS_UIO_H
#include <sys/uio.h>
#endif
#ifdef __MINGW32__
#define WINDOWS
#undef USE_SYSLOG
#endif /* __MINGW32__ */
#ifdef __CYGWIN__
/* Untested so far ... */
#define WINDOWS
#endif
#define RECEIVER_PORT(x) (x)
#define SENDER_PORT(x) ((x) + 1)
#define loseSendPacket udpc_loseSendPacket
#define loseRecvPacket udpc_loseRecvPacket
#define setWriteLoss udpc_setWriteLoss
#define setReadLoss udpc_setReadLoss
#define setReadSwap udpc_setReadSwap
#define srandomTime udpc_srandomTime
#define RecvMsg udpc_RecvMsg
#define doAutoRateLimit udpc_doAutoRateLimit
#define makeSockAddr udpc_makeSockAddr
#define printMyIp udpc_printMyIp
#define getSendBuf udpc_getSendBuf
#define setIpFromString udpc_setIpFromString
#define parseSize udpc_parseSize
#ifdef LOSSTEST
int loseSendPacket(void);
void loseRecvPacket(int s);
void setWriteLoss(char* l);
void setReadLoss(char* l);
void setReadSwap(char* l);
void srandomTime(int printSeed);
int RecvMsg(int s, struct msghdr* msg, int flags);
#endif
struct net_if
{
struct in_addr addr;
struct in_addr bcast;
const char* name;
#ifdef SIOCGIFINDEX
int index;
#endif
};
typedef struct net_if net_if_t;
typedef enum addr_type_t
{
ADDR_TYPE_UCAST,
ADDR_TYPE_MCAST,
ADDR_TYPE_BCAST
} addr_type_t;
void doAutoRateLimit(int sock, int dir, int qsize, int size);
int makeSockAddr(char* hostname, short port, struct sockaddr_in* addr);
void printMyIp(net_if_t* net_if);
int getSendBuf(int sock);
#define SEND(s, msg, to) doSend(s, &msg, sizeof(msg), &to)
#define RECV(s, msg, from, portBase) doReceive((s), &msg, sizeof(msg), &from, (portBase))
#define BCAST_CONTROL(s, msg) doSend(s, &msg, sizeof(msg), &net_config->controlMcastAddr)
void setIpFromString(struct sockaddr_in* addr, char* ip);
unsigned long parseSize(char* sizeString);
int udpc_socklibFatal(int code);
#ifdef __MINGW32__ /* __MINGW32__ */
struct iovec
{
void* iov_base;
int iov_len;
};
struct msghdr
{
void* msg_name;
int msg_namelen;
struct iovec* msg_iov;
int msg_iovlen;
};
ssize_t sendmsg(int s, const struct msghdr* msg, int flags);
ssize_t recvmsg(int fd, struct msghdr* msg, int flags);
#define usleep(x) Sleep((x) / 1000)
#define sleep(x) Sleep(1000L * (x))
#endif /* __MINGW32__ */
static inline void initMsgHdr(struct msghdr* hdr)
{
#ifndef WINDOWS
hdr->msg_control = 0;
hdr->msg_controllen = 0;
hdr->msg_flags = 0;
#endif
}
#ifndef __MINGW32__
#undef closesocket
#define closesocket(x) close(x)
#endif
#ifndef HAVE_IN_ADDR_T
typedef unsigned long in_addr_t;
#endif

36
utils/multicast/statistics.h
View File

@@ -1,36 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
typedef struct receiver_stats* receiver_stats_t;
typedef struct sender_stats* sender_stats_t;
#define allocReadStats udpc_allocReadStats
#define receiverStatsStartTimer udpc_receiverStatsStartTimer
#define displayReceiverStats udpc_displayReceiverStats
receiver_stats_t udpc_allocReadStats(int fd, long statPeriod, int printUncompressedPos);
void udpc_receiverStatsStartTimer(receiver_stats_t);
void udpc_displayReceiverStats(receiver_stats_t, int isFinal);
#define allocSenderStats udpc_allocSenderStats
#define displaySenderStats udpc_displaySenderStats
sender_stats_t udpc_allocSenderStats(int fd, FILE* logfile, long bwPeriod, long statPeriod,
int printUncompressedPos);
void udpc_displaySenderStats(sender_stats_t, int blockSize, int sliceSize, int isFinal);

141
utils/multicast/threads.h
View File

@@ -1,141 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#ifdef __MINGW32__
#include "socklib.h"
#include <winsock2.h>
#include <winbase.h>
#include <sys/time.h>
typedef HANDLE pthread_t;
typedef CRITICAL_SECTION pthread_mutex_t;
typedef HANDLE pthread_cond_t;
struct timespec
{
unsigned long tv_sec;
unsigned long tv_nsec;
};
static inline int pthread_create(pthread_t* thread, void* dummy1, LPTHREAD_START_ROUTINE start_routine,
void* arg)
{
/* Start thread ...
* see http://msdn.microsoft.com/library/default.asp?url=/library/en-us/dllproc/base/createthread.asp
*/
*thread = CreateThread(NULL, /* lpThreadAttributes */
0, /* dwStackSize */
start_routine, arg, /* lpParameter */
0, /* dwCreationFlags */
NULL /* lpThreadId */);
return *thread != NULL ? 0 : -1;
}
static inline int pthread_join(pthread_t th, void** thread_return)
{
return WaitForSingleObject(th, INFINITE) == WAIT_OBJECT_0 ? 0 : -1;
}
static inline int pthread_mutex_init(pthread_mutex_t* mutex, void* dummy)
{
InitializeCriticalSection(mutex);
return 0;
}
static inline int pthread_mutex_lock(pthread_mutex_t* mutex)
{
EnterCriticalSection(mutex);
return 0;
}
static inline int pthread_mutex_unlock(pthread_mutex_t* mutex)
{
LeaveCriticalSection(mutex);
return 0;
}
static inline int pthread_cond_init(pthread_cond_t* cond, void* dummy)
{
*cond = CreateEvent(NULL, TRUE, TRUE, NULL);
if (*cond == NULL)
return -1;
else
return 0;
}
static inline int pthread_cond_signal(pthread_cond_t* cond)
{
return SetEvent(*cond) ? 0 : -1;
}
static inline int pthread_cond_wait(pthread_cond_t* cond, pthread_mutex_t* mutex)
{
int r;
ResetEvent(*cond);
LeaveCriticalSection(mutex);
r = WaitForSingleObject(*cond, INFINITE) == WAIT_OBJECT_0 ? 0 : -1;
EnterCriticalSection(mutex);
return r;
}
static inline void pthread_cancel(pthread_t* thread)
{
TerminateThread(thread, 0);
}
#define ETIMEDOUT -2
#define MILLION 1000000
#define BILLION 1000000000
static inline int pthread_cond_timedwait(pthread_cond_t* cond, pthread_mutex_t* mutex, struct timespec* ts)
{
int r;
struct timeval tv;
long delta;
gettimeofday(&tv, NULL);
delta = (ts->tv_sec - tv.tv_sec) * 1000 + (ts->tv_nsec / BILLION - tv.tv_usec / MILLION);
if (delta < 0)
delta = 0;
ResetEvent(*cond);
LeaveCriticalSection(mutex);
switch (WaitForSingleObject(*cond, delta))
{
case WAIT_OBJECT_0: r = 0; break;
case WAIT_TIMEOUT: r = ETIMEDOUT; break;
default: r = -1; break;
}
EnterCriticalSection(mutex);
return r;
}
#define THREAD_RETURN DWORD WINAPI
#else /* __MINGW32__ */
#include <pthread.h>
#define THREAD_RETURN void*
#endif /* __MINGW32__ */

100
utils/multicast/udp-sender.h
View File

@@ -1,100 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "udp-sender.h"
#include "udpcast.h"
#include "participants.h"
#include "statistics.h"
#include "socklib.h"
extern FILE* udpc_log;
struct fifo;
#define openFile udpc_openFile
#define openPipe udpcs_openPipe
#define localReader udpc_localReader
#define doSend udpc_doSend
int openFile(struct disk_config* config);
int openPipe(struct disk_config* config, int in, int* pid);
int localReader(struct fifo* fifo, int in);
#define BCAST_DATA(s, msg) doSend(s, &msg, sizeof(msg), &net_config->dataMcastAddr)
/**
* "switched network" mode: server already starts sending next slice before
* first one is acknowledged. Do not use on old coax networks
*/
#define FLAG_SN 0x0001
/**
* "not switched network" mode: network is known not to be switched
*/
#define FLAG_NOTSN 0x0002
/**
* Asynchronous mode: do not any confirmation at all from clients.
* Useful in situations where no return channel is available
*/
#define FLAG_ASYNC 0x0004
/**
* Point-to-point transmission mode: use unicast in the (frequent)
* special case where there is only one receiver.
*/
#define FLAG_POINTOPOINT 0x0008
/**
* Do automatic rate limitation by monitoring socket's send buffer
* size. Not very useful, as this still doesn't protect against the
* switch dropping packets because its queue (which might be slightly slower)
* overruns
*/
#ifndef WINDOWS
#define FLAG_AUTORATE 0x0008
#endif
#ifdef BB_FEATURE_UDPCAST_FEC
/**
* Forward error correction
*/
#define FLAG_FEC 0x0010
#endif
/**
* Use broadcast rather than multicast (useful for cards that don't support
* multicast correctly
*/
#define FLAG_BCAST 0x0020
/**
* Never use point-to-point, even if only one receiver
*/
#define FLAG_NOPOINTOPOINT 0x0040
/*
* Don't ask for keyboard input on sender end.
*/
#define FLAG_NOKBD 0x0080
/**
* Streaming mode: allow receiver to join a running transmission
*/
#define FLAG_STREAMING 0x0100

204
utils/multicast/udpc-protoc.h
View File

@@ -1,204 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "udpcast.h"
#define MAX_BLOCK_SIZE 1456
#define MAX_FEC_INTERLEAVE 256
/**
* This file describes the UDPCast protocol
*/
enum opCode
{
/* Receiver to sender */
CMD_OK, /* all is ok, no need to retransmit anything */
CMD_RETRANSMIT, /* receiver asks for some data to be retransmitted */
CMD_GO, /* receiver tells server to start */
CMD_CONNECT_REQ, /* receiver tries to find out server's address */
CMD_DISCONNECT, /* receiver wants to disconnect itself */
CMD_UNUSED, /* obsolete version of CMD_HELLO, dating back to the
* time when we had little endianness (PC). This
* opcode contained a long unnoticed bug with parsing of
* blocksize */
/* Sender to receiver */
CMD_REQACK, /* server request acknowledgments from receiver */
CMD_CONNECT_REPLY, /* receiver tries to find out server's address */
CMD_DATA, /* a block of data */
CMD_FEC, /* a forward-error-correction block */
CMD_HELLO_NEW, /* sender says he's up */
CMD_HELLO_STREAMING, /* retransmitted hello during streaming mode */
};
/* Sender says he's up. This is not in the enum with the others,
* because of some endianness Snafu in early versions. However,since
* 2005-12-23, new receivers now understand a CMD_HELLO_NEW which is
* in sequence. Once enough of those are out in the field, we'll send
* CMD_HELLO_NEW by default, and then phase out the old variant. */
/* Tried to remove this on 2009-08-30, but noticed that receiver was printing
* "unexpected opcode" on retransmitted hello */
#define CMD_HELLO 0x0500
struct connectReq
{
unsigned short opCode;
short reserved;
int capabilities;
unsigned int rcvbuf;
};
struct retransmit
{
unsigned short opCode;
short reserved;
int sliceNo;
int rxmit;
unsigned char map[MAX_SLICE_SIZE / BITS_PER_CHAR];
};
struct ok
{
unsigned short opCode;
short reserved;
int sliceNo;
} ok;
union message
{
unsigned short opCode;
struct ok ok;
struct retransmit retransmit;
struct connectReq connectReq;
struct go
{
unsigned short opCode;
short reserved;
} go;
struct disconnect
{
unsigned short opCode;
short reserved;
} disconnect;
};
struct connectReply
{
unsigned short opCode;
short reserved;
int clNr;
int blockSize;
int capabilities;
unsigned char mcastAddr[16]; /* provide enough place for IPV6 */
};
struct hello
{
unsigned short opCode;
short reserved;
int capabilities;
unsigned char mcastAddr[16]; /* provide enough place for IPV6 */
short blockSize;
};
union serverControlMsg
{
unsigned short opCode;
short reserved;
struct hello hello;
struct connectReply connectReply;
};
struct dataBlock
{
unsigned short opCode;
short reserved;
int sliceNo;
unsigned short blockNo;
unsigned short reserved2;
int bytes;
};
struct fecBlock
{
unsigned short opCode;
short stripes;
int sliceNo;
unsigned short blockNo;
unsigned short reserved2;
int bytes;
};
struct reqack
{
unsigned short opCode;
short reserved;
int sliceNo;
int bytes;
int rxmit;
};
union serverDataMsg
{
unsigned short opCode;
struct reqack reqack;
struct dataBlock dataBlock;
struct fecBlock fecBlock;
};
/* ============================================
* Capabilities
*/
/* Does the receiver support the new CMD_DATA command, which carries
* capabilities mask?
* "new generation" receiver:
* - capabilities word included in hello/connectReq commands
* - receiver multicast capable
* - receiver can receive ASYNC and SN
*/
#define CAP_NEW_GEN 0x0001
/* Use multicast instead of Broadcast for data */
/*#define CAP_MULTICAST 0x0002*/
#ifdef BB_FEATURE_UDPCAST_FEC
/* Forward error correction */
#define CAP_FEC 0x0004
#endif
/* Supports big endians (a.k.a. network) */
#define CAP_BIG_ENDIAN 0x0008
/* Support little endians (a.k.a. PC) ==> obsolete! */
#define CAP_LITTLE_ENDIAN 0x0010
/* This transmission is asynchronous (no receiver reply) */
#define CAP_ASYNC 0x0020
/* Sender currently supports CAPABILITIES and MULTICAST */
#define SENDER_CAPABILITIES (CAP_NEW_GEN | CAP_BIG_ENDIAN)
#define RECEIVER_CAPABILITIES (CAP_NEW_GEN | CAP_BIG_ENDIAN)

157
utils/multicast/udpcast.h
View File

@@ -1,157 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "socklib.h"
#include <sys/time.h>
#include <stdio.h>
#include <pthread.h>
#define BITS_PER_INT (sizeof(int) * 8)
#define BITS_PER_CHAR 8
#define MAP_ZERO(l, map) (memset(map, 0, ((l) + BITS_PER_INT - 1) / BIT_PER_INT))
#define BZERO(data) (memset((void*)&data, 0, sizeof(data)))
#define RDATABUFSIZE (2 * (MAX_SLICE_SIZE + 1) * MAX_BLOCK_SIZE)
#define DATABUFSIZE (RDATABUFSIZE + 4096 - RDATABUFSIZE % 4096)
int udpc_writeSize(void);
int udpc_largeReadSize(void);
int udpc_smallReadSize(void);
int udpc_makeDataBuffer(int blocksize);
int udpc_parseCommand(char* pipeName, char** arg);
int udpc_printLongNum(unsigned long long x);
int udpc_waitForProcess(int pid, const char* message);
struct disk_config
{
int origOutFile;
const char* fileName;
char* pipeName;
int flags;
struct timeval stats_last_printed;
};
#define MAX_GOVERNORS 10
struct net_config
{
net_if_t* net_if; /* Network interface (eth0, isdn0, etc.) on which to
* multicast */
int portBase; /* Port base */
int blockSize;
int sliceSize;
struct sockaddr_in controlMcastAddr;
struct sockaddr_in dataMcastAddr;
const char* mcastRdv;
int ttl;
int nrGovernors;
struct rateGovernor_t* rateGovernor[MAX_GOVERNORS];
void* rateGovernorData[MAX_GOVERNORS];
/*int async;*/
/*int pointopoint;*/
struct timeval ref_tv;
enum discovery
{
DSC_DOUBLING,
DSC_REDUCING
} discovery;
/* int autoRate; do queue watching using TIOCOUTQ, to avoid overruns */
int flags; /* non-capability command line flags */
int capabilities;
int min_slice_size;
int default_slice_size;
int max_slice_size;
unsigned int rcvbuf;
int rexmit_hello_interval; /* retransmission interval between hello's.
* If 0, hello message won't be retransmitted
*/
int autostart; /* autostart after that many retransmits */
int requestedBufSize; /* requested receiver buffer */
/* sender-specific parameters */
int min_receivers;
int max_receivers_wait;
int min_receivers_wait;
int retriesUntilDrop;
/* receiver-specif parameters */
int exitWait; /* How many milliseconds to wait on program exit */
int startTimeout; /* Timeout at start */
/* FEC config */
#ifdef BB_FEATURE_UDPCAST_FEC
int fec_redundancy; /* how much fec blocks are added per group */
int fec_stripesize; /* size of FEC group */
int fec_stripes; /* number of FEC stripes per slice */
#endif
int rehelloOffset; /* how far before end will rehello packet will
be retransmitted */
};
struct stat_config
{
FILE* log; /* Log file for statistics */
long bwPeriod; /* How often are bandwidth estimations logged? */
int statPeriod;
int printUncompressedPos;
};
#define NR_CLIENT_SOCKS 4
struct client_config
{
int socks[NR_CLIENT_SOCKS];
struct sockaddr_in serverAddr;
int clientNumber;
int isStarted;
pthread_t thread;
int sender_is_newgen;
};
void* rgInitGovernor(struct net_config* cfg, struct rateGovernor_t* gov);
void rgParseRateGovernor(struct net_config* net_config, char* rg);
void rgWaitAll(struct net_config* cfg, int sock, in_addr_t ip, int size);
void rgShutdownAll(struct net_config* cfg);
/**
* Answers whether given fd is seekable
*/
int udpc_shouldPrintUncompressedPos(int deflt, int fd, int pipe);
#define MAX_SLICE_SIZE 1024
#define DEFLT_STAT_PERIOD 500000
#ifndef DEBUG
#define DEBUG 0
#endif

30
utils/multicast/util.h
View File

@@ -1,30 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include <stdlib.h>
#define MALLOC(type) ((type*)calloc(1, sizeof(type)))
/* bitmap manipulation */
#define BITS_PER_ITEM(map) (sizeof(map[0]) * 8)
#define MASK(pos, map) (1 << ((pos) % (BITS_PER_ITEM(map))))
#define POS(pos, map) ((pos) / BITS_PER_ITEM(map))
#define SET_BIT(x, map) (map[POS(x, map)] |= MASK(x, map))
#define CLR_BIT(x, map) (map[POS(x, map)] &= ~MASK(x, map))
#define BIT_ISSET(x, map) (map[POS(x, map)] & MASK(x, map))

7
utils/pron/CMakeLists.txt
View File

@@ -1,6 +1,5 @@
include_directories(${ENGINE_COMMON_INCLUDES})
set(pron_LIB_SRCS pron.cpp)
add_library(pron STATIC ${pron_LIB_SRCS})
target_link_libraries(pron messageqcpp loggingcpp)
# We don't isntall static library install(TARGETS pron DESTINATION ${ENGINE_LIBDIR} COMPONENT columnstore-engine)
set(pron_LIB_SRCS pron.cpp)
columnstore_static_library(pron ${pron_LIB_SRCS})
columnstore_link(pron PRIVATE messageqcpp loggingcpp)

12
utils/querystats/CMakeLists.txt
View File

@@ -3,16 +3,6 @@ add_definitions(-DMYSQL_SERVICE_THD_TIMEZONE_INCLUDED)
include_directories(${ENGINE_COMMON_INCLUDES})
# ########## next target ###############
set(querystats_LIB_SRCS querystats.cpp)
columnstore_library(querystats ${querystats_LIB_SRCS})
add_dependencies(querystats loggingcpp)
install(
TARGETS querystats
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(querystats PRIVATE loggingcpp)

13
utils/querytele/CMakeLists.txt
View File

@@ -1,18 +1,11 @@
include_directories(${ENGINE_COMMON_INCLUDES})
# ########## next target ###############
set(querytele_LIB_SRCS querytele.cpp queryteleclient.cpp querytele_constants.cpp querytele_types.cpp
QueryTeleService.cpp queryteleprotoimpl.cpp
)
columnstore_library(querytele ${querytele_LIB_SRCS})
add_dependencies(querytele external_boost external_thrift)
columnstore_link(querytele ${THRIFT_LIBRARY})
target_include_directories(querytele PRIVATE ${THRIFT_INCLUDE_DIRS})
target_link_libraries(querytele ${THRIFT_LIBRARY})
install(
TARGETS querytele
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
add_dependencies(querytele external_boost external_thrift)

17
utils/regr/CMakeLists.txt
View File

@@ -21,24 +21,11 @@ set(regr_LIB_SRCS
add_definitions(-DMYSQL_DYNAMIC_PLUGIN)
columnstore_library(regr ${regr_LIB_SRCS})
add_dependencies(regr loggingcpp)
install(
TARGETS regr
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(regr PRIVATE loggingcpp)
set(regr_mysql_LIB_SRCS regrmysql.cpp modamysql.cpp)
columnstore_library(regr_mysql ${regr_mysql_LIB_SRCS})
columnstore_mysql_plugin_library(regr_mysql SHARED ${regr_mysql_LIB_SRCS})
add_dependencies(regr_mysql external_boost)
install(
TARGETS regr_mysql
DESTINATION ${MARIADB_PLUGINDIR}
COMPONENT columnstore-engine
)
set_target_properties(regr_mysql PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../../)

14
utils/rowgroup/CMakeLists.txt
View File

@@ -4,16 +4,6 @@ include_directories(${ENGINE_COMMON_INCLUDES})
set(rowgroup_LIB_SRCS rowaggregation.cpp rowgroup.cpp rowstorage.cpp)
# librowgroup_la_CXXFLAGS = $(march_flags) $(AM_CXXFLAGS)
columnstore_library(rowgroup ${rowgroup_LIB_SRCS})
add_dependencies(rowgroup loggingcpp external_boost)
target_link_libraries(rowgroup ${NETSNMP_LIBRARIES} funcexp)
install(
TARGETS rowgroup
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(rowgroup PRIVATE ${NETSNMP_LIBRARIES} funcexp loggingcpp)
add_dependencies(rowgroup external_boost)

9
utils/rwlock/CMakeLists.txt
View File

@@ -1,12 +1,5 @@
include_directories(${ENGINE_COMMON_INCLUDES})
set(rwlock_LIB_SRCS rwlock.cpp rwlock_local.cpp)
columnstore_library(rwlock ${rwlock_LIB_SRCS})
add_dependencies(rwlock external_boost)
install(
TARGETS rwlock
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(rwlock boost_thread)

7
utils/startup/CMakeLists.txt
View File

@@ -1,8 +1,5 @@
include_directories(${ENGINE_COMMON_INCLUDES})
add_definitions(-fPIC -DPIC)
add_library(idbboot STATIC installdir.cpp)
set(idbboot_LIB_SRCS installdir.cpp)
columnstore_static_library(idbboot ${idbboot_LIB_SRCS})
add_dependencies(idbboot external_boost)
install(TARGETS idbboot DESTINATION ${ENGINE_LIBDIR})

97
utils/testbc/blockcacheclient.cpp
View File

@@ -1,97 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***************************************************************************
*
* $Id: blockcacheclient.cpp 684 2008-08-19 22:22:59Z pleblanc $
*
* jrodriguez@calpont.com *
***************************************************************************/
#include <sstream>
#include <string>
#include "blockcacheclient.h"
namespace dbbc
{
blockCacheClient::blockCacheClient()
{
fBCCBrp = NULL;
}
blockCacheClient::blockCacheClient(BlockRequestProcessor& brp)
{
fBCCBrp = &brp;
}
blockCacheClient::~blockCacheClient()
{
}
void blockCacheClient::check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache)
{
fBCCBrp->check(lbid, ver, flg, wasBlockInCache);
}
void blockCacheClient::check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& rCount)
{
fBCCBrp->check(range, ver, rCount);
}
FileBuffer* blockCacheClient::getBlockPtr(const BRM::LBID_t& lbid, const BRM::VER_t& ver)
{
FileBuffer* fb = fBCCBrp->getBlockPtr(lbid, ver);
return fb;
}
const int blockCacheClient::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb)
{
int ret = fBCCBrp->read(lbid, ver, fb);
return ret;
}
const int blockCacheClient::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr)
{
int ret = fBCCBrp->read(lbid, ver, bufferPtr);
return ret;
}
const int blockCacheClient::read(const BRM::InlineLBIDRange& range, FileBufferList_t& fbList,
const BRM::VER_t ver)
{
int ret = fBCCBrp->read(range, fbList, ver);
return ret;
}
const int blockCacheClient::getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr,
bool flg, bool& wasCached)
{
int ret = fBCCBrp->getBlock(lbid, ver, bufferPtr, flg, wasCached);
return ret;
}
bool blockCacheClient::exists(BRM::LBID_t lbid, BRM::VER_t ver)
{
return fBCCBrp->exists(lbid, ver);
}
void blockCacheClient::flushCache()
{
fBCCBrp->flushCache();
}
} // namespace dbbc

104
utils/testbc/blockcacheclient.h
View File

@@ -1,104 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***************************************************************************
*
* $Id: blockcacheclient.h 684 2008-08-19 22:22:59Z pleblanc $
*
* *
***************************************************************************/
#pragma once
#include "blockrequestprocessor.h"
#include "brmtypes.h"
/**
@author Jason Rodriguez <jrodriguez@calpont.com>
*/
/**
* @brief API for the Disk Block Buffer Cache
*
*
*/
namespace dbbc
{
class blockCacheClient
{
public:
/**
* @brief ctor requires reference to BlockRequestProcessor object
**/
blockCacheClient(BlockRequestProcessor& brp);
/**
* @brief dtor
**/
virtual ~blockCacheClient();
/**
* @brief verify that the Disk Block for the LBID lbid, ver are loaded into the Cache.
**/
void check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache);
/**
* @brief verify all Disk Blocks for the LBID range are loaded into the Cache
**/
void check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& rCount);
/**
* @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache
**/
const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb);
FileBuffer* getBlockPtr(const BRM::LBID_t& lbid, const BRM::VER_t& ver);
/**
* @brief retrieve the Disk Block at lbid, ver from the Disk Block Buffer Cache
**/
const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr);
/**
* @brief retrieve all disk Blocks in the LBIDRange range and insert them into fbList
**/
const int read(const BRM::InlineLBIDRange& range, FileBufferList_t& fbList, const BRM::VER_t ver);
const int getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr, bool flg,
bool& wasCached);
bool exists(BRM::LBID_t lbid, BRM::VER_t ver);
/**
* @brief flush the cache
**/
void flushCache();
private:
/**
* @brief pointer to the BlockRequestProcessor object on which the API will operate
**/
BlockRequestProcessor* fBCCBrp;
// do not implement
blockCacheClient();
blockCacheClient(const blockCacheClient& bc);
const blockCacheClient& operator=(const blockCacheClient& blk);
};
} // namespace dbbc

227
utils/testbc/blockrequestprocessor.cpp
View File

@@ -1,227 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***************************************************************************
*
* $Id: blockrequestprocessor.cpp 725 2008-09-26 16:26:47Z jrodriguez $
*
* jrodriguez@calpont.com *
* *
***************************************************************************/
#include "blockrequestprocessor.h"
#include "rwlock_local.h"
#include "dbrm.h"
#include <sys/time.h>
#include <pthread.h>
#include <sstream>
#include <string>
#include <list>
#include <boost/date_time/posix_time/posix_time.hpp>
using namespace std;
namespace dbbc
{
BlockRequestProcessor::BlockRequestProcessor(uint32_t numBlcks, int thrCount, int blocksPerRead,
uint32_t deleteBlocks, uint32_t blckSz)
: fbMgr(numBlcks, blckSz, deleteBlocks), fIOMgr(fbMgr, fBRPRequestQueue, thrCount, blocksPerRead)
{
}
BlockRequestProcessor::~BlockRequestProcessor()
{
}
void BlockRequestProcessor::stop()
{
fBRPRequestQueue.stop();
fIOMgr.stop();
}
int BlockRequestProcessor::check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& lbidCount)
{
uint64_t maxLbid = range.start; // highest existent lbid
uint64_t rangeLen = range.size;
uint64_t idx;
uint64_t adjSz;
struct timespec start_tm;
if (fTrace)
clock_gettime(CLOCK_MONOTONIC, &start_tm);
for (idx = 0; fbMgr.exists(maxLbid, ver) == true && idx < rangeLen; maxLbid++, idx++)
;
if (idx == rangeLen) // range is already loaded
{
uint32_t fbo;
BRM::OID_t oid;
fdbrm.lookup(maxLbid, ver, false, oid, fbo);
fLogFile << oid << " " << maxLbid << " " << fbo << " " << rangeLen << " " << 0 << " " << 0 << " " << 0
<< " " << right << fixed << ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec))) << endl;
return 0;
}
adjSz = rangeLen - idx;
BRM::InlineLBIDRange adjRange;
adjRange.start = maxLbid;
adjRange.size = adjSz;
fileRequest rqstBlk(adjRange, ver);
check(rqstBlk);
lbidCount = rqstBlk.BlocksRead();
if (fTrace)
{
uint32_t fbo;
BRM::OID_t oid;
fdbrm.lookup(maxLbid, ver, false, oid, fbo);
fLogFile << oid << " " << maxLbid << " " << fbo << " " << rangeLen << " " << adjSz << " "
<< rqstBlk.BlocksRead() << " " << rqstBlk.BlocksLoaded() << " " << right << fixed
<< ((double)(start_tm.tv_sec + (1.e-9 * start_tm.tv_nsec))) << endl;
}
return rqstBlk.BlocksLoaded();
} // check
int BlockRequestProcessor::check(fileRequest& rqstBlk)
{
pthread_mutex_lock(&rqstBlk.frMutex());
rqstBlk.SetPredicate(fileRequest::SENDING);
sendRequest(rqstBlk); // start file read request
while (rqstBlk.frPredicate() < fileRequest::COMPLETE)
pthread_cond_wait(&rqstBlk.frCond(), &rqstBlk.frMutex());
pthread_mutex_unlock(&rqstBlk.frMutex());
return 0;
}
int BlockRequestProcessor::check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache)
{
if (fbMgr.exists(lbid, ver) == true)
{
wasBlockInCache = true;
return 0;
}
else
{
wasBlockInCache = false;
fileRequest rqstBlk(lbid, ver, flg);
int ret = check(rqstBlk);
return ret;
}
}
int BlockRequestProcessor::sendRequest(fileRequest& blk)
{
int ret = fBRPRequestQueue.push(blk);
return ret;
}
const int BlockRequestProcessor::read(const BRM::InlineLBIDRange& range, FileBufferList_t& readList,
const BRM::VER_t ver)
{
int blksLoaded = 0;
HashObject_t fb = {0, 0, 0};
for (int idx = 0; (uint64_t)idx < range.size; idx++)
{
fb.lbid = range.start + idx;
fb.ver = ver;
fb.poolIdx = 0;
FileBuffer fbRet(-1, -1);
bool ret = false; // fbMgr.find(fb, fbRet);
if (ret)
{
blksLoaded++;
readList.push_back(fbRet);
}
}
return blksLoaded;
}
FileBuffer* BlockRequestProcessor::getBlockPtr(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
HashObject_t hashObj = {lbid, ver, 0};
FileBuffer* fb = fbMgr.findPtr(hashObj);
return fb;
}
const int BlockRequestProcessor::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb)
{
HashObject_t hashObj = {lbid, ver, 0};
bool ret = fbMgr.find(hashObj, fb);
if (ret == true)
return 1;
else
return 0;
}
const int BlockRequestProcessor::read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr)
{
HashObject_t hashObj = {lbid, ver, 0};
bool ret = fbMgr.find(hashObj, bufferPtr);
if (ret == true)
return 1;
else
return 0;
}
const int BlockRequestProcessor::getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr,
bool flg, bool& wasCached)
{
HashObject_t hashObj = {lbid, ver, 0};
wasCached = fbMgr.find(hashObj, bufferPtr);
if (wasCached)
return 1;
wasCached = false;
fileRequest rqstBlk(lbid, ver, flg, (uint8_t*)bufferPtr);
check(rqstBlk);
return 1;
}
bool BlockRequestProcessor::exists(BRM::LBID_t lbid, BRM::VER_t ver)
{
HashObject_t ho = {lbid, ver, 0};
return fbMgr.exists(ho);
}
void BlockRequestProcessor::flushCache()
{
fbMgr.flushCache();
}
/**
const uint32_t BlockRequestProcessor::resize(const uint32_t s)
{
int rc = fbMgr.resize(s);
return rc;
}
**/
ostream& BlockRequestProcessor::formatLRUList(ostream& os) const
{
return fbMgr.formatLRUList(os);
}
} // namespace dbbc

131
utils/testbc/blockrequestprocessor.h
View File

@@ -1,131 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
/***************************************************************************
*
* $Id: blockrequestprocessor.h 725 2008-09-26 16:26:47Z jrodriguez $
*
* jrodriguez@calpont.com *
* *
***************************************************************************/
#include "blocksize.h"
#include "fileblockrequestqueue.h"
#include "filebuffermgr.h"
#include "iomanager.h"
/**
@author Jason Rodriguez <jrodriguez@calpont.com>
*/
namespace dbbc
{
typedef std::list<FileBuffer> FileBufferList_t;
/**
* @brief class to control the populating of the Disk Block Buffer Cache and manage Block requests.
**/
class BlockRequestProcessor
{
public:
/**
* @brief default ctor
**/
BlockRequestProcessor(uint32_t numBlcks, int thrCount, int blocksPerRead, uint32_t deleteBlocks = 0,
uint32_t blckSz = BLOCK_SIZE);
/**
* @brief default dtor
**/
virtual ~BlockRequestProcessor();
/**
* @brief send a request for disk blocks to the IO manager
**/
int sendRequest(fileRequest& blk);
/**
* @brief verify that the lbid@ver disk block is in the block cache. Send request if it is not
**/
int check(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, bool& wasBlockInCache);
/**
* @brief verify the LBIDRange of disk blocks is in the block cache. Send request if it is not
**/
int check(const BRM::InlineLBIDRange& range, const BRM::VER_t ver, uint32_t& lbidCount);
/**
* @brief retrieve the lbid@ver disk block from the block cache
**/
FileBuffer* getBlockPtr(const BRM::LBID_t lbid, const BRM::VER_t ver);
const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, FileBuffer& fb);
/**
* @brief retrieve the lbid@ver disk block from the block cache
**/
const int read(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr);
/**
* @brief retrieve the LBIDRange of disk blocks from the block cache
**/
const int read(const BRM::InlineLBIDRange& range, FileBufferList_t& fbList, const BRM::VER_t ver);
const int getBlock(const BRM::LBID_t& lbid, const BRM::VER_t& ver, void* bufferPtr, bool flg,
bool& wasCached);
bool exists(BRM::LBID_t lbid, BRM::VER_t ver);
/**
* @brief
**/
void flushCache();
// const uint32_t resize(const uint32_t s);
std::ostream& formatLRUList(std::ostream& os) const;
private:
FileBufferMgr fbMgr;
fileBlockRequestQueue fBRPRequestQueue;
ioManager fIOMgr;
pthread_mutex_t check_mutex;
/**
* helper function for public check functions
**/
int check(fileRequest& rqstBlk);
/**
* send stop requests for IOmanager and request Q
**/
void stop();
std::ofstream fLogFile;
bool fTrace;
BRM::DBRM fdbrm;
// do not implement
BlockRequestProcessor(const BlockRequestProcessor& brp);
BlockRequestProcessor& operator=(const BlockRequestProcessor& brp);
};
} // namespace dbbc

92
utils/testbc/fileblockrequestqueue.cpp
View File

@@ -1,92 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***************************************************************************
*
* $Id: fileblockrequestqueue.cpp 666 2008-07-22 14:39:46Z wweeks $
*
* jrodriguez@calpont.com *
* *
***************************************************************************/
#include "fileblockrequestqueue.h"
using namespace std;
namespace dbbc
{
fileBlockRequestQueue::fileBlockRequestQueue() : queueSize(0), readersWaiting(0)
{
pthread_mutex_init(&mutex, NULL);
pthread_cond_init(&notEmpty, NULL);
}
fileBlockRequestQueue::~fileBlockRequestQueue()
{
pthread_cond_destroy(&notEmpty);
pthread_mutex_destroy(&mutex);
}
bool fileBlockRequestQueue::empty() const
{
return (queueSize == 0);
}
fileRequest* fileBlockRequestQueue::top() const
{
return fbQueue.front();
}
int fileBlockRequestQueue::push(fileRequest& blk)
{
pthread_mutex_lock(&mutex);
fbQueue.push_back(&blk);
// @bug 1007. Changed "== 1" to ">= 1" below. The wake up call was only being fired when the queue size
// was 1 which caused only one i/o thread to be working at a time.
if (++queueSize >= 1 && readersWaiting > 0)
pthread_cond_signal(&notEmpty);
pthread_mutex_unlock(&mutex);
return 0;
}
void fileBlockRequestQueue::stop()
{
pthread_cond_broadcast(&notEmpty);
}
fileRequest* fileBlockRequestQueue::pop(void)
{
pthread_mutex_lock(&mutex);
while (queueSize == 0)
{
readersWaiting++;
pthread_cond_wait(&notEmpty, &mutex);
readersWaiting--;
}
fileRequest* blk = fbQueue.front();
fbQueue.pop_front();
--queueSize;
pthread_mutex_unlock(&mutex);
return blk;
}
} // namespace dbbc

108
utils/testbc/fileblockrequestqueue.h
View File

@@ -1,108 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
/***************************************************************************
*
* $Id: fileblockrequestqueue.h 483 2008-02-10 20:23:28Z rdempsey $
*
* jrodriguez@calpont.com *
* *
***************************************************************************/
#include <deque>
#include <pthread.h>
#include <iostream>
#include "filerequest.h"
/**
@author Jason Rodriguez <jrodriguez@calpont.com>
*/
/**
* @brief definition of the block request queue as stl std::priority_queue
**/
namespace dbbc
{
typedef std::deque<fileRequest*> fileBlockRequestQueue_t;
/**
* @brief class to hold requests for disk blocks in a queue. sorted by the size of a request
**/
class fileBlockRequestQueue
{
public:
/**
* @brief default ctor
**/
fileBlockRequestQueue();
/**
* @brief dtor
**/
virtual ~fileBlockRequestQueue();
/**
* @brief add a request to the queue
**/
int push(fileRequest& blk);
/**
* @brief get the next request from the queue and delete it from the queue
**/
fileRequest* pop(void);
/**
* @brief true if no reuquests are in the queue. false if there are requests in the queue
**/
bool empty() const;
/**
* @brief number of requests in the queue
**/
uint32_t size() const
{
return queueSize;
}
/**
* @brief queue will stop accecpting requests in preparation for the dtor
**/
void stop();
protected:
pthread_mutex_t mutex;
pthread_cond_t notEmpty;
fileBlockRequestQueue_t fbQueue;
uint32_t queueSize;
uint32_t readersWaiting;
private:
// do not implement
fileBlockRequestQueue(const fileBlockRequestQueue& Q)
{
}
const fileBlockRequestQueue& operator=(const fileBlockRequestQueue& Q);
/**
* @brief pointer to the next request to be popped from the queue
**/
fileRequest* top() const;
};
} // namespace dbbc

85
utils/testbc/filebuffer.cpp
View File

@@ -1,85 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***************************************************************************
*
* $Id: filebuffer.cpp 643 2008-06-30 16:39:59Z jrodriguez $
*
* jrodriguez@calpont.com *
* *
***************************************************************************/
#include "filebuffer.h"
#include <sstream>
#include <string>
#include <iostream>
using namespace std;
namespace dbbc
{
FileBuffer::FileBuffer(const FileBuffer& rhs)
{
if (this == NULL || this == &rhs)
return;
fLbid = rhs.fLbid;
fVerid = rhs.fVerid;
setData(rhs.fByteData, rhs.fDataLen);
fListLoc = rhs.listLoc();
fDataLen = rhs.fDataLen;
}
FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len)
{
fLbid = lbid;
fVerid = ver;
fDataLen = len;
setData(data, fDataLen);
}
FileBuffer::FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
fLbid = lbid;
fVerid = ver;
fDataLen = 0;
}
FileBuffer& FileBuffer::operator=(const FileBuffer& rhs)
{
fLbid = rhs.fLbid;
fVerid = rhs.fVerid;
fDataLen = rhs.fDataLen;
setData(rhs.fByteData, fDataLen);
fListLoc = rhs.listLoc();
return *this;
}
void FileBuffer::setData(const uint8_t* d, const int len)
{
if (d == NULL || len <= 0)
return;
fDataLen = len;
memcpy(fByteData, d, len);
}
FileBuffer::~FileBuffer()
{
}
} // namespace dbbc

182
utils/testbc/filebuffer.h
View File

@@ -1,182 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***************************************************************************
*
* $Id: filebuffer.h 643 2008-06-30 16:39:59Z jrodriguez $
*
* jrodriguez@calpont.com *
* *
***************************************************************************/
#pragma once
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <stdexcept>
#include <stdint.h>
#include <time.h>
#include <we_define.h>
#include "brmtypes.h"
#include <list>
#include <vector>
/**
@author Jason Rodriguez <jrodriguez@calpont.com>
*/
/**
* @brief represents a disk blockrequest
**/
namespace dbbc
{
// Set block cache alogorithm to last recently used by defining LRU.
// Otherwise it will be FIFO.
typedef struct FBData
{
BRM::LBID_t lbid;
BRM::VER_t ver;
} FBData_t;
//@bug 669 Change to list for last recently used cache
typedef std::list<FBData_t> filebuffer_list_t;
typedef std::list<FBData_t>::iterator filebuffer_list_iter_t;
class FileBuffer
{
public:
/**
* @brief copy ctor
**/
FileBuffer(const FileBuffer& fb);
/**
* @brief the disk block from lbid@ver, and a data block len bytes long
**/
FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data, const uint32_t len);
/**
* @brief disk block lbid@ver and empty data
**/
FileBuffer(const BRM::LBID_t lbid, const BRM::VER_t ver);
/**
* @brief class dtor
**/
~FileBuffer();
/**
* @brief set the data value of this block to d have len bytestream
**/
void setData(const uint8_t* d, const int len = 8192);
/**
* @brief retrieve the data in byte* format from this data block
**/
const uint8_t* getData() const
{
return fByteData;
}
uint8_t* getData()
{
return fByteData;
}
const uint32_t datLen() const
{
return fDataLen;
}
/**
* @brief assignment operator
**/
FileBuffer& operator=(const FileBuffer& rhs);
/**
* @brief equality operator is based on lbid@ver
**/
bool operator==(const FileBuffer& rhs) const
{
return (fLbid == rhs.fLbid && fVerid == rhs.fVerid);
}
/**
* @brief inequality operator
**/
bool operator!=(const FileBuffer& rhs) const
{
return (!(fLbid == rhs.fLbid && fVerid == rhs.fVerid));
}
FileBuffer* thisPtr()
{
return this;
}
/**
* @brief return the lbid value of disk bloc
**/
const BRM::LBID_t Lbid() const
{
return fLbid;
}
void Lbid(const BRM::LBID_t l)
{
fLbid = l;
}
/**
* @brief return the version of this disk block. ignored for range retrievals
**/
const BRM::VER_t Verid() const
{
return fVerid;
}
void Verid(BRM::VER_t v)
{
fVerid = v;
}
/**
* @brief return the number of bytes in this disk blockrequest
**/
void listLoc(const filebuffer_list_iter_t& loc)
{
fListLoc = loc;
}
const filebuffer_list_iter_t& listLoc() const
{
return fListLoc;
}
private:
uint8_t fByteData[WriteEngine::BYTE_PER_BLOCK];
uint32_t fDataLen;
BRM::LBID_t fLbid;
BRM::VER_t fVerid;
filebuffer_list_iter_t fListLoc;
// do not implement
FileBuffer(){};
};
typedef std::vector<FileBuffer> FileBufferPool_t;
} // namespace dbbc

299
utils/testbc/filebuffermgr.cpp
View File

@@ -1,299 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***************************************************************************
*
* $Id: filebuffermgr.cpp 699 2008-09-09 19:44:18Z rdempsey $
*
* jrodriguez@calpont.com *
* *
***************************************************************************/
/**
* InitialDBBCSize - the starting number of elements the unordered set used to store disk blocks.
This does not instantiate InitialDBBCSize disk blocks but only the initial size of the unordered_set
**/
#define NDEBUG // Turn off assert macro
#include <cassert>
#include <limits.h>
#include <boost/thread.hpp>
#include "filebuffermgr.h"
using namespace boost;
using namespace std;
#include "stats.h"
extern dbbc::Stats* gPMStatsPtr;
extern bool gPMProfOn;
extern uint32_t gSession;
namespace dbbc
{
FileBufferMgr::FileBufferMgr(const uint32_t numBlcks, const uint32_t blkSz, const uint32_t deleteBlocks)
: fMaxNumBlocks(numBlcks)
, fBlockSz(blkSz)
, fWLock()
, fbSet()
, fbList()
, fCacheSize(0)
, fFBPool()
, aging(false)
, fDeleteBlocks(deleteBlocks)
, fEmptyPoolSlots()
{
fFBPool.reserve(numBlcks);
fEmptyPoolSlots.reserve(deleteBlocks);
}
FileBufferMgr::~FileBufferMgr()
{
flushCache();
}
void FileBufferMgr::flushCache()
{
mutex::scoped_lock lk(fWLock);
fbList.clear();
fbSet.clear();
fFBPool.clear();
fEmptyPoolSlots.clear();
// TODO:: re-init blocks in pool and HWM
}
bool FileBufferMgr::exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const
{
const HashObject_t fb = {lbid, ver, 0};
const bool b = exists(fb);
return b;
} // bool FileBufferMgr::exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const
FileBuffer* FileBufferMgr::findPtr(const HashObject_t& keyFb)
{
mutex::scoped_lock lk(fWLock);
filebuffer_uset_iter_t it = fbSet.find(keyFb);
if (fbSet.end() != it)
{
FileBuffer* fb = &(fFBPool[it->poolIdx]);
fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc());
return fb;
}
return NULL;
} // end findPtr(const HashObject_t& keyFB)
bool FileBufferMgr::find(const HashObject_t& keyFb, FileBuffer& fb)
{
bool ret = false;
mutex::scoped_lock lk(fWLock);
filebuffer_uset_iter_t it = fbSet.find(keyFb);
if (fbSet.end() != it)
{
fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc());
lk.unlock();
fb = fFBPool[it->poolIdx];
ret = true;
}
return ret;
} // end find(const HashObject_t& keyFB, HashObject_t& fb)
bool FileBufferMgr::find(const HashObject_t& keyFb, void* bufferPtr)
{
bool ret = false;
if (gPMProfOn && gPMStatsPtr)
gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'L');
mutex::scoped_lock lk(fWLock);
if (gPMProfOn && gPMStatsPtr)
gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'M');
filebuffer_uset_iter_t it = fbSet.find(keyFb);
if (fbSet.end() != it)
{
//@bug 669 LRU cache, move block to front of list as last recently used.
fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc());
lk.unlock();
memcpy(bufferPtr, (fFBPool[it->poolIdx]).getData(), 8);
if (gPMProfOn && gPMStatsPtr)
gPMStatsPtr->markEvent(keyFb.lbid, pthread_self(), gSession, 'U');
ret = true;
}
return ret;
} // end find(const FileBuffer& keyFB, void* bufferPtr)
bool FileBufferMgr::exists(const HashObject_t& fb) const
{
bool find_bool = false;
mutex::scoped_lock lk(fWLock);
filebuffer_uset_iter_t it = fbSet.find(fb);
if (it != fbSet.end())
{
find_bool = true;
fbList.splice(fbList.begin(), fbList, (fFBPool[it->poolIdx]).listLoc());
}
return find_bool;
}
// default insert operation.
// add a new fb into fbMgr and to fbList
// add to the front and age out from the back
// so add new fbs to the front of the list
//@bug 665: keep filebuffer in a vector. HashObject keeps the index of the filebuffer
const int FileBufferMgr::insert(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data)
{
int ret = 0;
if (gPMProfOn && gPMStatsPtr)
gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'I');
mutex::scoped_lock lk(fWLock);
HashObject_t fbIndex = {lbid, ver, 0};
filebuffer_pair_t pr = fbSet.insert(fbIndex);
if (pr.second)
{
// It was inserted (it wasn't there before)
// Right now we have an invalid cache: we have inserted an entry with a -1 index.
// We need to fix this quickly...
fCacheSize++;
FBData_t fbdata = {lbid, ver};
fbList.push_front(fbdata);
}
else
{
// if it's a duplicate there's nothing to do
if (gPMProfOn && gPMStatsPtr)
gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'D');
return ret;
}
uint32_t pi = INT_MAX;
if (fCacheSize > maxCacheSize())
{
// If the insert above caused the cache to exceed its max size, find the lru block in
// the cache and use its pool index to store the block data.
FBData_t& fbdata = fbList.back(); // the lru block
HashObject_t lastFB = {fbdata.lbid, fbdata.ver, 0};
filebuffer_uset_iter_t iter = fbSet.find(lastFB); // should be there
idbassert(iter != fbSet.end());
pi = iter->poolIdx;
idbassert(pi < maxCacheSize());
idbassert(pi < fFBPool.size());
/* Why does this iterator return a const HashObject_t? */
HashObject_t& ref = const_cast<HashObject_t&>(*pr.first);
ref.poolIdx = pi;
// replace the lru block with this block
FileBuffer fb(lbid, ver, NULL, 0);
fFBPool[pi] = fb;
fFBPool[pi].setData(data, BLOCK_SIZE);
fbSet.erase(iter);
fbList.pop_back();
fCacheSize--;
depleteCache();
ret = 1;
}
else
{
if (!fEmptyPoolSlots.empty())
{
pi = fEmptyPoolSlots.back();
fEmptyPoolSlots.pop_back();
FileBuffer fb(lbid, ver, NULL, 0);
fFBPool[pi] = fb;
fFBPool[pi].setData(data, 8);
}
else
{
pi = fFBPool.size();
FileBuffer fb(lbid, ver, NULL, 0);
fFBPool.push_back(fb);
fFBPool[pi].setData(data, 8);
}
/* Why does this iterator return a const? */
HashObject_t& ref = const_cast<HashObject_t&>(*pr.first);
ref.poolIdx = pi;
ret = 1;
}
idbassert(pi < fFBPool.size());
fFBPool[pi].listLoc(fbList.begin());
if (gPMProfOn && gPMStatsPtr)
gPMStatsPtr->markEvent(lbid, pthread_self(), gSession, 'J');
idbassert(fCacheSize <= maxCacheSize());
// idbassert(fCacheSize == fbSet.size());
// idbassert(fCacheSize == fbList.size());
return ret;
}
void FileBufferMgr::depleteCache()
{
for (uint32_t i = 0; i < fDeleteBlocks && !fbList.empty(); ++i)
{
FBData_t fbdata(fbList.back()); // the lru block
HashObject_t lastFB = {fbdata.lbid, fbdata.ver, 0};
filebuffer_uset_iter_t iter = fbSet.find(lastFB);
idbassert(iter != fbSet.end());
uint32_t idx = iter->poolIdx;
idbassert(idx < fFBPool.size());
// Save position in FileBuffer pool for reuse.
fEmptyPoolSlots.push_back(idx);
fbSet.erase(iter);
fbList.pop_back();
fCacheSize--;
}
}
ostream& FileBufferMgr::formatLRUList(ostream& os) const
{
filebuffer_list_t::const_iterator iter = fbList.begin();
filebuffer_list_t::const_iterator end = fbList.end();
while (iter != end)
{
os << iter->lbid << '\t' << iter->ver << endl;
++iter;
}
return os;
}
} // namespace dbbc

197
utils/testbc/filebuffermgr.h
View File

@@ -1,197 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***************************************************************************
*
* $Id: filebuffermgr.h 699 2008-09-09 19:44:18Z rdempsey $
*
* *
***************************************************************************/
#pragma once
#include <pthread.h>
#include "blocksize.h"
#include "filebuffer.h"
#include "rwlock_local.h"
#include <tr1/unordered_set>
#include <boost/thread.hpp>
/**
@author Jason Rodriguez <jrodriguez@calpont.com>
*/
/**
* @brief manages storage of Disk Block Buffers via and LRU cache using the stl classes unordered_set and
*list.
*
**/
namespace dbbc
{
/**
* @brief used as the hasher algorithm for the unordered_set used to store the disk blocks
**/
typedef struct
{
BRM::LBID_t lbid;
BRM::VER_t ver;
uint32_t poolIdx;
} FileBufferIndex_t;
typedef FileBufferIndex_t HashObject_t;
class bcHasher
{
public:
size_t operator()(const HashObject_t& rhs) const
{
return (((rhs.ver & 0xffffULL) << 48) | (rhs.lbid & 0xffffffffffffULL));
}
};
class bcEqual
{
public:
size_t operator()(const HashObject_t& f1, const HashObject_t& f2) const
{
return ((f1.lbid == f2.lbid) && (f1.ver == f2.ver));
}
};
inline bool operator<(const HashObject_t& f1, const HashObject_t& f2)
{
// return ((f1.lbid < f2.lbid) || (f1.ver < f2.ver));
#if 1
if (f1.lbid < f2.lbid)
return true;
else if (f1.lbid == f2.lbid)
return (f1.ver < f2.ver);
return false;
#else
bcHasher bh1, bh2;
return (bh1(f1) < bh2(f2));
#endif
}
class FileBufferMgr
{
public:
typedef std::tr1::unordered_set<HashObject_t, bcHasher, bcEqual> filebuffer_uset_t;
typedef std::tr1::unordered_set<HashObject_t, bcHasher, bcEqual>::const_iterator filebuffer_uset_iter_t;
typedef std::pair<filebuffer_uset_t::iterator, bool> filebuffer_pair_t; // return type for insert
typedef std::vector<uint32_t> intvec_t;
/**
* @brief ctor. Set max buffer size to numBlcks and block buffer size to blckSz
**/
FileBufferMgr(uint32_t numBlcks, uint32_t blckSz = BLOCK_SIZE, uint32_t deleteBlocks = 0);
/**
* @brief default dtor
**/
virtual ~FileBufferMgr();
/**
* @brief return TRUE if the Disk block lbid@ver is loaded into the Disk Block Buffer cache otherwise return
*FALSE.
**/
bool exists(const BRM::LBID_t& lbid, const BRM::VER_t& ver) const;
/**
* @brief return TRUE if the Disk block referenced by fb is loaded into the Disk Block Buffer cache
*otherwise return FALSE.
**/
bool exists(const HashObject_t& fb) const;
/**
* @brief add the Disk Block reference by fb into the Disk Block Buffer Cache
**/
const int insert(const BRM::LBID_t lbid, const BRM::VER_t ver, const uint8_t* data);
/**
* @brief returns the total number of Disk Blocks in the Cache
**/
uint32_t size() const
{
return fbSet.size();
}
/**
* @brief
**/
void flushCache();
/**
* @brief return the disk Block referenced by fb
**/
FileBuffer* findPtr(const HashObject_t& keyFb);
bool find(const HashObject_t& keyFb, FileBuffer& fb);
/**
* @brief return the disk Block referenced by bufferPtr
**/
bool find(const HashObject_t& keyFb, void* bufferPtr);
uint32_t maxCacheSize() const
{
return fMaxNumBlocks;
}
uint32_t listSize() const
{
return fbList.size();
}
const filebuffer_uset_iter_t end() const
{
return fbSet.end();
}
void displayCounts() const;
std::ostream& formatLRUList(std::ostream& os) const;
private:
uint32_t fMaxNumBlocks; // the max number of blockSz blocks to keep in the Cache list
uint32_t fBlockSz; // size in bytes size of a data block - probably 8
mutable boost::mutex fWLock;
mutable filebuffer_uset_t fbSet;
mutable filebuffer_list_t fbList; // rename this
uint32_t fCacheSize;
FileBufferPool_t fFBPool; // vector<FileBuffer>
// do not implement
FileBufferMgr(const FileBufferMgr& fbm);
const FileBufferMgr& operator=(const FileBufferMgr& fbm);
bool aging;
uint32_t fDeleteBlocks;
intvec_t fEmptyPoolSlots; // keep track of FBPool slots that can be reused
void depleteCache();
};
} // namespace dbbc

98
utils/testbc/filerequest.cpp
View File

@@ -1,98 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/***************************************************************************
*
* $Id: filerequest.cpp 506 2008-03-14 15:31:55Z jrodriguez $
*
* jrodriguez@calpont.com *
* *
***************************************************************************/
#include "filerequest.h"
#include <stdexcept>
using namespace std;
namespace dbbc
{
fileRequest::fileRequest()
{
fLBID = -1;
fVer = -1;
data = NULL;
init();
fRqstType = LBIDREQUEST;
}
fileRequest::fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg)
: data(NULL), fLBID(lbid), fVer(ver), fFlg(flg)
{
init();
fLength = 1;
fRqstType = LBIDREQUEST;
}
fileRequest::fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, uint8_t* ptr)
: fLBID(lbid), fVer(ver), fFlg(flg)
{
init();
fLength = 1;
fRqstType = LBIDREQUEST;
data = ptr;
}
fileRequest::fileRequest(const BRM::InlineLBIDRange& range, const BRM::VER_t ver)
: data(NULL), fLBID(range.start), fVer(ver), fFlg(false), fLength(range.size), fRqstType(RANGEREQUEST)
{
init();
fLength = range.size;
}
fileRequest::fileRequest(const fileRequest& blk)
{
fLBID = blk.fLBID;
fVer = blk.fVer;
fLength = blk.fLength;
fblksRead = blk.fblksRead;
fRqstType = blk.fRqstType;
fRqstStatus = blk.fRqstStatus;
data = blk.data;
init();
}
void fileRequest::init()
{
if (pthread_mutex_init(&fFRMutex, NULL) != 0)
throw runtime_error("mutex initialization failure");
if (pthread_cond_init(&fFRCond, NULL) != 0)
throw runtime_error("cond var initialization failure");
fFRPredicate = INIT;
fLength = 0;
fblksRead = 0;
fRqstStatus = 0;
}
fileRequest::~fileRequest()
{
pthread_mutex_destroy(&fFRMutex);
pthread_cond_destroy(&fFRCond);
}
} // namespace dbbc

252
utils/testbc/filerequest.h
View File

@@ -1,252 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
/***************************************************************************
*
* $Id: filerequest.h 658 2008-07-10 14:47:24Z jrodriguez $
*
* jrodriguez@calpont.com *
* *
***************************************************************************/
#include "brmtypes.h"
/**
@author Jason Rodriguez <jrodriguez@calpont.com>
*/
/**
* @brief request class for the fileblockrequestqueue and the iomanager
**/
namespace dbbc
{
class fileRequest
{
public:
/**
* @brief default ctor
**/
fileRequest();
/**
* @brief copy constructor
**/
fileRequest(const fileRequest& blk);
/**
* @brief request for the disk block lbid@ver
**/
fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg);
fileRequest(BRM::LBID_t lbid, BRM::VER_t ver, bool flg, uint8_t* ptr);
/**
* @brief request a range of disk blocks
**/
fileRequest(const BRM::InlineLBIDRange& range, const BRM::VER_t ver);
/**
* @brief class dtor
**/
virtual ~fileRequest();
/**
* @brief less-than operator
**/
bool operator<(const fileRequest& rhs) const
{
return fLength < rhs.fLength;
}
/**
* @brief greater-than operator
**/
bool operator>(const fileRequest& rhs) const
{
return fLength > rhs.fLength;
}
/**
* @brief equality operator
**/
bool operator==(const fileRequest& rhs) const
{
return fLength == rhs.fLength;
}
enum request_status_enum
{
SUCCESSFUL,
FAILED
};
enum request_type_enum
{
LBIDREQUEST,
RANGEREQUEST
};
/**
* @brief used to manage request processing synchronzation
**/
enum predicate_status_enum
{
INIT,
SENDING,
READING,
COMPLETE,
STOP
};
/**
* @brief lbid requested
**/
const BRM::LBID_t Lbid() const
{
return fLBID;
}
/**
* @brief version of the lbid requested
**/
const BRM::VER_t Ver() const
{
return fVer;
}
/**
* @brief VBBM flag of the LBID/Ver
**/
const bool Flg() const
{
return fFlg;
}
/**
* @brief number of blocks requested
**/
const uint32_t BlocksRequested() const
{
return fLength;
}
/**
* @brief setter for blocks requested
**/
void BlocksRequested(const int l)
{
fLength = l;
}
/**
* @brief number of blocks read from disk
**/
const uint32_t BlocksRead() const
{
return fblksRead;
}
const uint32_t BlocksLoaded() const
{
return fblksLoaded;
}
/**
* @brief setter for blocks read from disk
**/
void BlocksRead(const int l)
{
fblksRead = l;
}
void BlocksLoaded(const int l)
{
fblksLoaded = l;
}
/**
* @brief did the request succeed for fail
**/
int RequestStatus() const
{
return fRqstStatus;
}
/**
* @brief setter for the request status
**/
void RequestStatus(int s)
{
fRqstStatus = s;
}
/**
* @brief return BLOCK or RANGE requested
**/
int RequestType() const
{
return fRqstType;
}
/**
* @brief mutex to control synchronzation of request processing
**/
pthread_mutex_t& frMutex() const
{
return fFRMutex;
}
/**
* @brief condition variable. signal when request is complete
**/
pthread_cond_t& frCond() const
{
return fFRCond;
}
/**
* @brief
**/
const enum predicate_status_enum& frPredicate() const
{
return fFRPredicate;
}
/**
* @brief setter for the predicate
**/
void SetPredicate(const enum predicate_status_enum& p)
{
fFRPredicate = p;
}
uint8_t* data;
private:
void init();
BRM::LBID_t fLBID;
BRM::VER_t fVer;
bool fFlg;
mutable pthread_mutex_t fFRMutex;
mutable pthread_cond_t fFRCond;
predicate_status_enum fFRPredicate;
uint32_t fLength; // lbids requested
uint32_t fblksRead; // lbids read
uint32_t fblksLoaded; // lbids loaded into cache
int fRqstStatus;
enum request_type_enum fRqstType;
};
} // namespace dbbc

327
utils/testbc/iomanager.cpp
View File

@@ -1,327 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: iomanager.cpp 724 2008-09-26 16:16:05Z jrodriguez $
//
// C++ Implementation: iomanager
//
// Description:
//
//
// Author: Jason Rodriguez <jrodriguez@calpont.com>
//
//
//
#define _FILE_OFFSET_BITS 64
#define _LARGEFILE64_SOURCE
#include <stdexcept>
#include <iostream>
#include <iomanip>
#include <unistd.h>
#include <stdlib.h>
#include <string>
#include <sstream>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <errno.h>
#include <boost/shared_ptr.hpp>
#include <boost/scoped_array.hpp>
#define NDEBUG
#include <cassert>
using namespace std;
#include "configcpp.h"
using namespace config;
#include "iomanager.h"
namespace
{
using namespace dbbc;
using namespace std;
void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, double& tm)
{
tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec);
}
struct IOMThreadArg
{
ioManager* iom;
int32_t thdId;
};
typedef IOMThreadArg IOMThreadArg_t;
void* thr_popper(void* arg)
{
ioManager* iom = ((IOMThreadArg*)arg)->iom;
int32_t iomThdId = ((IOMThreadArg*)arg)->thdId;
FileBufferMgr* fbm;
int totalRqst = 0;
fileRequest* fr = 0;
BRM::LBID_t lbid = 0;
BRM::OID_t oid = 0;
BRM::VER_t ver = 0;
int blocksLoaded = 0;
int blocksRead = 0;
const unsigned pageSize = 4096;
fbm = &iom->fileBufferManager();
char fileName[WriteEngine::FILE_NAME_SIZE];
const uint64_t fileBlockSize = BLOCK_SIZE;
uint32_t offset = 0;
bool flg = false;
char* fileNamePtr = fileName;
uint64_t longSeekOffset = 0;
int err;
uint32_t dlen = 0, acc, readSize, blocksThisRead, j;
uint32_t blocksRequested = 0;
ssize_t i;
uint32_t sz = 0;
char* alignedbuff = 0;
boost::scoped_array<char> realbuff;
pthread_t threadId = 0;
ostringstream iomLogFileName;
ofstream lFile;
threadId = pthread_self();
uint32_t readBufferSz = iom->blocksPerRead * BLOCK_SIZE + pageSize;
realbuff.reset(new char[readBufferSz]);
if (realbuff.get() == 0)
{
cerr << "thr_popper: Can't allocate space for a whole extent in memory" << endl;
return 0;
}
#if __WORDSIZE > 32
// alignedbuff=(char*)((((ptrdiff_t)&realbuff[0] + pageSize - 1) / pageSize) * pageSize);
// pagesize == (1 << 12)
alignedbuff = (char*)((((ptrdiff_t)realbuff.get() >> 12) << 12) + pageSize);
#else
// alignedbuff=(char*)(((((ptrdiff_t)&realbuff[0] & 0xffffffff) + pageSize - 1) / pageSize) * pageSize);
alignedbuff = (char*)(((((ptrdiff_t)realbuff.get() >> 12) << 12) & 0xffffffff) + pageSize);
#endif
idbassert(((ptrdiff_t)alignedbuff - (ptrdiff_t)realbuff.get()) < (ptrdiff_t)pageSize);
idbassert(((ptrdiff_t)alignedbuff % pageSize) == 0);
for (;;)
{
fr = iom->getNextRequest();
lbid = fr->Lbid();
ver = fr->Ver();
flg = fr->Flg();
blocksLoaded = 0;
blocksRead = 0;
dlen = fr->BlocksRequested();
blocksRequested = fr->BlocksRequested();
err = iom->lbidLookup(lbid, ver, flg, oid, offset);
if (err < 0)
{
cerr << "lbid=" << lbid << " ver=" << ver << " flg=" << (flg ? 1 : 0) << endl;
throw runtime_error("thr_popper: BRM lookup failure");
}
longSeekOffset = (uint64_t)offset * (uint64_t)fileBlockSize;
totalRqst++;
sz = 0;
uint32_t readCount = 0;
uint32_t bytesRead = 0;
uint32_t jend = blocksRequested / iom->blocksPerRead;
for (j = 0; j <= jend; j++)
{
blocksThisRead = std::min(dlen, iom->blocksPerRead);
readSize = blocksThisRead * BLOCK_SIZE;
acc = 0;
while (acc < readSize)
{
i = readSize; // pread(fd, &alignedbuff[acc], readSize - acc, longSeekOffset);
/* XXXPAT: Need to decide how to handle errors here */
if (i < 0 && errno == EINTR)
{
continue;
}
else if (i < 0)
{
perror("thr_popper::read");
return 0; // shuts down this thread,
// probably not the right thing to do
}
else if (i == 0)
{
try
{
}
catch (exception& exc)
{
cerr << "FileName Err:" << exc.what() << endl;
}
cerr << "thr_popper: Early EOF in file " << fileNamePtr << endl;
return 0;
}
acc += i;
longSeekOffset += (uint64_t)i;
readCount++;
bytesRead += i;
} // while(acc...
blocksRead += blocksThisRead;
for (i = 0; (unsigned)i < blocksThisRead; ++i)
{
if (fbm->insert((lbid + i) + (j * iom->blocksPerRead), ver, (uint8_t*)&alignedbuff[i * BLOCK_SIZE]))
++blocksLoaded;
}
dlen -= blocksThisRead;
} // for (j...
fr->BlocksRead(blocksRead);
fr->BlocksLoaded(blocksLoaded);
if (fr->data != 0 && blocksRequested == 1)
memcpy(fr->data, alignedbuff, BLOCK_SIZE);
pthread_mutex_lock(&fr->frMutex());
fr->SetPredicate(fileRequest::COMPLETE);
pthread_cond_signal(&fr->frCond());
pthread_mutex_unlock(&fr->frMutex());
} // for(;;)
lFile.close();
return 0;
} // end thr_popper
} // anonymous namespace
namespace dbbc
{
ioManager::ioManager(FileBufferMgr& fbm, fileBlockRequestQueue& fbrq, int thrCount, int bsPerRead)
: blocksPerRead(bsPerRead), fIOMfbMgr(fbm), fIOMRequestQueue(fbrq)
{
if (thrCount <= 0)
thrCount = 1;
if (thrCount > 256)
thrCount = 256;
fConfig = Config::makeConfig();
string val = fConfig->getConfig("DBBC", "IOMTracing");
int temp = 0;
if (val.length() > 0)
temp = static_cast<int>(Config::fromText(val));
if (temp > 0)
fIOTrace = true;
else
fIOTrace = false;
fThreadCount = thrCount;
go();
} // ioManager
void ioManager::buildOidFileName(const BRM::OID_t oid, char* file_name)
{
if (fFileOp.getFileName(oid, file_name) != WriteEngine::NO_ERROR)
{
file_name[0] = 0;
throw std::runtime_error("fileOp.getFileName failed");
}
}
BRM::LBID_t ioManager::lbidLookup(BRM::LBID_t lbid, BRM::VER_t verid, bool vbFlag, BRM::OID_t& oid,
uint32_t& offset)
{
int rc = fdbrm.lookup(lbid, verid, vbFlag, oid, offset);
return rc;
}
int ioManager::createReaders()
{
int realCnt = 0;
IOMThreadArg_t fThdArgArr[256];
for (int idx = 0; idx < fThreadCount; idx++)
{
fThdArgArr[realCnt].iom = this;
fThdArgArr[realCnt].thdId = realCnt;
int ret = pthread_create(&fThreadArr[realCnt], 0, thr_popper, &fThdArgArr[realCnt]);
if (ret != 0)
perror("createReaders::pthread_create");
else
realCnt++;
}
fThreadCount = realCnt;
return fThreadCount;
}
ioManager::~ioManager()
{
stop();
}
void ioManager::go(void)
{
createReaders();
}
void ioManager::stop()
{
for (int idx = 0; idx < fThreadCount; idx++)
{
pthread_detach(fThreadArr[idx]);
}
}
fileRequest* ioManager::getNextRequest()
{
fileRequest* blk = 0;
try
{
blk = fIOMRequestQueue.pop();
return blk;
}
catch (exception& e)
{
cerr << "ioManager::getNextRequest() ERROR " << endl;
}
return blk;
}
} // namespace dbbc

100
utils/testbc/iomanager.h
View File

@@ -1,100 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
// $Id: iomanager.h 655 2008-07-08 16:42:54Z jrodriguez $
//
// C++ Interface: iomanager
//
// Description:
//
//
// Author: Jason Rodriguez <jrodriguez@calpont.com>
//
//
//
/**
@author Jason Rodriguez <jrodriguez@calpont.com>
*/
#include <pthread.h>
#include <string>
#include "writeengine.h"
#include "configcpp.h"
#include "brm.h"
#include "fileblockrequestqueue.h"
#include "filebuffermgr.h"
namespace dbbc
{
class ioManager
{
public:
ioManager(FileBufferMgr& fbm, fileBlockRequestQueue& fbrq, int thrCount, int bsPerRead);
// ioManager(FileBufferMgr& fbm, int thrCount);
~ioManager();
int readerCount() const
{
return fThreadCount;
}
fileRequest* getNextRequest();
void go(void);
void stop();
FileBufferMgr& fileBufferManager()
{
return fIOMfbMgr;
}
config::Config* configPtr()
{
return fConfig;
}
BRM::LBID_t lbidLookup(BRM::LBID_t lbid, BRM::VER_t verid, bool vbFlag, BRM::OID_t& oid, uint32_t& offset);
void buildOidFileName(const BRM::OID_t oid, char* file_name);
uint32_t getExtentSize()
{
return fdbrm.getExtentSize();
}
uint32_t blocksPerRead;
bool IOTrace() const
{
return fIOTrace;
}
private:
FileBufferMgr& fIOMfbMgr;
fileBlockRequestQueue& fIOMRequestQueue;
int fThreadCount;
pthread_t fPoppertid;
pthread_t fThreadArr[256];
int createReaders();
config::Config* fConfig;
BRM::DBRM fdbrm;
WriteEngine::FileOp fFileOp;
// do not implement
ioManager();
ioManager(const ioManager& iom);
const ioManager& operator=(const ioManager& iom);
bool fIOTrace;
};
} // namespace dbbc

63
utils/testbc/logger.cpp
View File

@@ -1,63 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*
* $Id: logger.cpp 686 2008-08-21 22:08:33Z rdempsey $
*/
#include <boost/thread.hpp>
using namespace boost;
#include "messageobj.h"
#include "messageids.h"
#include "loggingid.h"
using namespace logging;
#include "logger.h"
namespace primitiveprocessor
{
Logger::Logger() : fMl1(LoggingID(28))
{
fMsgMap[logging::M0000] = Message(logging::M0000);
fMsgMap[logging::M0016] = Message(logging::M0016);
fMsgMap[logging::M0045] = Message(logging::M0045);
fMsgMap[logging::M0053] = Message(logging::M0053);
}
void Logger::logMessage(const Message::MessageID mid, const Message::Args& args, bool critical)
{
mutex::scoped_lock lk(fLogLock);
MsgMap::iterator msgIter = fMsgMap.find(mid);
if (msgIter == fMsgMap.end())
msgIter = fMsgMap.find(logging::M0000);
msgIter->second.reset();
msgIter->second.format(args);
if (critical)
{
fMl1.logCriticalMessage(msgIter->second);
}
else
{
fMl1.logWarningMessage(msgIter->second);
}
}
} // namespace primitiveprocessor

62
utils/testbc/logger.h
View File

@@ -1,62 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*
* $Id: logger.h 706 2008-09-16 18:25:49Z bwelch $
*/
/** @file */
#pragma once
#include <map>
#include <boost/thread.hpp>
#include "messageobj.h"
#include "messagelog.h"
namespace primitiveprocessor
{
/** @brief message log wrapper class */
class Logger
{
public:
Logger();
void logMessage(const logging::Message::MessageID mid, const logging::Message::Args& args,
bool critical = false);
void logMessage(const std::string& msg, bool critical = true, logging::Message::MessageID mid = 0)
{
logging::Message::Args args;
args.add(msg);
logMessage(mid, args, true);
}
private:
// defaults okay
// Logger(const Logger& rhs);
// Logger& operator=(const Logger& rhs);
typedef std::map<logging::Message::MessageID, logging::Message> MsgMap;
MsgMap fMsgMap;
boost::mutex fLogLock;
logging::MessageLog fMl1;
};
} // namespace primitiveprocessor

273
utils/testbc/stats.cpp
View File

@@ -1,273 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
Copyright (C) 2016 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*
* $Id: stats.cpp 699 2008-09-09 19:44:18Z rdempsey $
*/
#include <ctime>
#include <sys/time.h>
#include <pthread.h>
#include <iomanip>
#include <map>
#define NDEBUG
#include <cassert>
#include <csignal>
using namespace std;
#include <boost/thread.hpp>
#include <boost/shared_ptr.hpp>
using namespace boost;
#include "stats.h"
#include "messagelog.h"
using namespace BRM;
namespace
{
void pause_(unsigned delay)
{
struct timespec req;
struct timespec rem;
req.tv_sec = delay;
req.tv_nsec = 0;
rem.tv_sec = 0;
rem.tv_nsec = 0;
again:
if (nanosleep(&req, &rem) != 0)
if (rem.tv_sec > 0 || rem.tv_nsec > 0)
{
req = rem;
goto again;
}
}
const string timestr()
{
// Get a timestamp for output.
struct tm tm;
struct timeval tv;
gettimeofday(&tv, 0);
localtime_r(&tv.tv_sec, &tm);
ostringstream oss;
oss << setfill('0') << setw(2) << tm.tm_hour << ':' << setw(2) << tm.tm_min << ':' << setw(2) << tm.tm_sec
<< '.' << setw(4) << tv.tv_usec / 100;
return oss.str();
}
class TraceFile
{
public:
TraceFile(uint32_t sessionID, const char* name)
{
if (sessionID > 0)
{
const char* outName;
if (name == 0)
outName = "lbids";
else
outName = name;
ostringstream oss;
oss << MCSLOGDIR << "/trace/" << outName << '.' << sessionID;
oFile.reset(new ofstream());
oFile->open(oss.str().c_str(), ios_base::out | ios_base::ate | ios_base::app);
}
}
~TraceFile()
{
}
void close()
{
if (oFile)
{
oFile->close();
}
}
void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event = '\0')
{
*oFile << oid << ' ' << timestr() << ' ' << lbid << ' ' << thdid;
if (event != '\0')
*oFile << ' ' << event;
*oFile << endl;
oFile->flush();
}
private:
// Compiler defaults okay
// TraceFile(const TraceFile& rhs);
// TraceFile operator=(const TraceFile& rhs);
shared_ptr<ofstream> oFile;
};
struct TraceFileInfo
{
TraceFileInfo(uint32_t session = 0, const char* name = 0) : traceFile(session, name), lastTouched(0)
{
}
~TraceFileInfo()
{
}
void log(OID_t oid, uint64_t lbid, pthread_t thdid, char event = '\0')
{
traceFile.log(oid, lbid, thdid, event);
lastTouched = time(0);
}
void close()
{
traceFile.close();
}
TraceFile traceFile;
time_t lastTouched;
private:
// Compiler defaults okay
// TraceFileInfo(const TraceFileInfo& rhs);
// TraceFileInfo operator=(const TraceFileInfo& rhs);
};
// map a session id to a trace file
typedef map<uint32_t, TraceFileInfo> TraceFileMap_t;
TraceFileMap_t traceFileMap;
// map mutex
mutex traceFileMapMutex;
class StatMon
{
public:
StatMon()
{
sigset_t sigset;
sigemptyset(&sigset);
sigaddset(&sigset, SIGPIPE);
sigaddset(&sigset, SIGUSR1);
sigaddset(&sigset, SIGUSR2);
pthread_sigmask(SIG_BLOCK, &sigset, 0);
}
void operator()() const
{
// struct timespec ts = { 60 * 1, 0 };
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter;
TraceFileMap_t::iterator end;
for (;;)
{
lk.unlock();
time_t beforeSleep = time(0);
// nanosleep(&ts, 0);
pause_(60);
lk.lock();
iter = traceFileMap.begin();
end = traceFileMap.end();
while (iter != end)
{
if (iter->second.lastTouched < beforeSleep)
{
// remove this session trace file
iter->second.close();
traceFileMap.erase(iter++);
}
else
++iter;
}
}
}
private:
// Compiler defaults okay
// StatMon(const StatMon& rhs);
// StatMon operator=(const StatMon& rhs);
};
} // namespace
namespace dbbc
{
Stats::Stats() : fMonitorp(0)
{
fMonitorp = new thread(StatMon());
}
Stats::Stats(const char* name) : fMonitorp(0), fName(name)
{
fMonitorp = new thread(StatMon());
// fName << name;
}
Stats::~Stats()
{
delete fMonitorp;
}
void Stats::touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session)
{
if (lbid < 0 || session == 0)
return;
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter = traceFileMap.find(session);
if (iter == traceFileMap.end())
{
traceFileMap[session] = TraceFileInfo(session);
iter = traceFileMap.find(session);
idbassert(iter != traceFileMap.end());
}
iter->second.log(lbid2oid(lbid), lbid, thdid);
}
void Stats::markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event)
{
if (lbid < 0 || session == 0)
return;
mutex::scoped_lock lk(traceFileMapMutex);
TraceFileMap_t::iterator iter = traceFileMap.find(session);
if (iter == traceFileMap.end())
{
traceFileMap[session] = TraceFileInfo(session, fName);
iter = traceFileMap.find(session);
idbassert(iter != traceFileMap.end());
}
iter->second.log(lbid2oid(lbid), lbid, thdid, event);
}
} // namespace dbbc

63
utils/testbc/stats.h
View File

@@ -1,63 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*
* $Id: stats.h 609 2008-06-11 12:40:07Z rdempsey $
*/
/** @file */
#pragma once
#include <pthread.h>
#include <boost/thread.hpp>
#include <iostream>
#include <sstream>
#include "brm.h"
namespace dbbc
{
class Stats
{
public:
Stats();
Stats(const char* name);
virtual ~Stats();
void touchedLBID(uint64_t lbid, pthread_t thdid, uint32_t session = 0);
void markEvent(const uint64_t lbid, const pthread_t thdid, const uint32_t session, const char event);
inline BRM::OID_t lbid2oid(uint64_t lbid)
{
BRM::OID_t oid;
uint32_t fbo;
brm.lookup(lbid, 0, false, oid, fbo);
return oid;
}
private:
Stats(const Stats& rhs);
Stats& operator=(const Stats& rhs);
boost::thread* fMonitorp;
BRM::DBRM brm;
// ostringstream fName;
const char* fName;
};
} // namespace dbbc

361
utils/testbc/testbc.cpp
View File

@@ -1,361 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
//
// C++ Implementation: bcTest
//
// Description: A simple Test driver for the Disk Block Buffer Cache
//
// Author: Jason Rodriguez <jrodriguez@calpont.com>, (C) 2007
//
//
#include <vector>
#include <string>
#include <iomanip>
#include <iostream>
#include <sys/time.h>
#include <unistd.h>
#include <boost/thread/thread.hpp>
#include "blockrequestprocessor.h"
#include "blockcacheclient.h"
#include "stats.h"
#include "brm.h"
#include "logger.h"
#include "iomanager.h"
using namespace BRM;
using namespace dbbc;
using namespace std;
using namespace logging;
using namespace primitiveprocessor;
Stats* gPMStatsPtr = NULL;
bool gPMProfOn = false;
uint32_t gSession = 0;
uint32_t lastRangeListIdx = 0;
void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, double& tm)
{
tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec);
}
namespace primitiveprocessor
{
Logger ml;
}
class BCTest
{
public:
struct OidRanges
{
OID_t oid;
HWM_t hwm;
LBIDRange_v ranges;
OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r)
{
oid = o;
hwm = h;
ranges = r;
}
}; // struct OidRanges
BCTest(const int cacheSz = 64 * 1024, int readThr = 2, int readAhead = 1024);
typedef OidRanges OidRanges_t;
typedef vector<OidRanges_t> OidRangesList_t;
OidRangesList_t OidRangesList;
DBRM dbrm;
uint32_t extentSize;
BRM::OID_t maxOid;
int fCacheSz;
int fReadThr;
int fReadAhead;
uint32_t maxBlocksAvailable;
uint32_t fExtentSize;
void setUp();
int LoadOid(const OidRanges_t& o, uint32_t& loadCount);
void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver);
int ReadOidRanges(const OidRanges_t& v);
void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver);
BlockRequestProcessor BRP;
}; // class BCTest
BCTest::BCTest(int cacheSz, int readThr, int readAhead)
: fCacheSz(cacheSz), fReadThr(readThr), fReadAhead(readAhead), BRP(fCacheSz, fReadThr, fReadAhead)
{
setUp();
} // ctor
//
void BCTest::setUp()
{
LBIDRange_v r;
HWM_t hwm;
OID_t oid = 1000;
extentSize = dbrm.getExtentSize();
maxBlocksAvailable = 0;
int i = 0;
fExtentSize = dbrm.getExtentSize();
while (oid < 5000)
{
int ret = 0;
ret = dbrm.lookup(oid, r);
if (ret == 0 && r.size() > 0)
{
dbrm.getHWM(oid, hwm);
maxBlocksAvailable += (r.size() * extentSize);
OidRanges_t oid_range(oid, hwm, r);
OidRangesList.push_back(oid_range);
// cout << "Setup i: " << i++ << " o: " << oid
// << " r: " << ret << " s: " << r.size()
// << " m: " << maxBlocksAvailable
// << endl;
hwm = 0;
r.clear();
}
oid++;
}
// cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl;
i = 0;
} // setUp()
int BCTest::LoadOid(const OidRanges_t& o, uint32_t& loadCount)
{
blockCacheClient bc(BRP);
uint32_t rCount = 0;
for (uint32_t i = 0; i < o.ranges.size(); i++)
{
const InlineLBIDRange r = {o.ranges[i].start, o.ranges[i].size};
if (r.size > 0)
{
bc.check(r, 0, rCount);
// cout << "i: " << i << " c: " << rCount << " " << o.ranges[i].size << endl;
loadCount += rCount;
}
rCount = 0;
} // for
// cout << "hwm: " << o.hwm << " tot: " << loadCount << " " << o.ranges.size() << endl;
return loadCount;
} // LoadOid
int BCTest::ReadOidRanges(const OidRanges_t& v)
{
blockCacheClient bc(BRP);
int32_t readBlocks = 0;
int32_t missBlocks = 0;
// int ret;
for (uint32_t i = 0; i < v.ranges.size(); i++)
{
FileBuffer fb(-1, -1);
const InlineLBIDRange r = {v.ranges[i].start, v.ranges[i].size};
for (int j = r.start; readBlocks < fCacheSz && j < r.start + r.size; j++)
{
// ret=0;
// ret=bc.read(j, 0, fb);
FileBuffer* ptr = bc.getBlockPtr(j, 0);
if (ptr)
readBlocks++;
else
missBlocks++;
}
// cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: "
// << v.hwm << endl;
}
return readBlocks;
} // ReadRange
// add one block to block cache
//
void BCTest::LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
blockCacheClient bc(BRP);
bool b;
bc.check(lbid, ver, false, b);
} // LoadLbid
// get one block out of block cache
//
void BCTest::ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
uint8_t d[8192] = {'\0'};
blockCacheClient bc(BRP);
bc.read(lbid, ver, d);
} // ReadLbid
struct loadThr
{
loadThr::loadThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps)
{
}
void operator()()
{
uint32_t loadedBlocks = 0;
uint32_t oidBlocks;
uint32_t i = 0;
uint32_t rc = 0;
clock_gettime(CLOCK_REALTIME, &tm1);
for (uint32_t j = 0; j < fReps; j++)
for (i = 0; /*loadedBlocks<cacheSize &&*/ i < fBC.OidRangesList.size(); i++)
{
oidBlocks = 0;
rc = fBC.LoadOid(fBC.OidRangesList[i], oidBlocks);
/**
cout << "."
<< "-- " << i << " " << fBC.OidRangesList[i].oid
<< " h: " << fBC.OidRangesList[i].hwm
<< "/" << oidBlocks
<< endl;
**/
loadedBlocks += oidBlocks;
}
clock_gettime(CLOCK_REALTIME, &tm2);
double tm3;
timespec_sub(tm1, tm2, tm3);
lastRangeListIdx = i;
cout << "loadtest ld: " << loadedBlocks << " sz: "
<< fBC.fCacheSz
//<< " last: " << lastRangeListIdx
<< " tm: " << right << setw(10) << fixed << tm3 << endl;
} // operator()
BCTest& fBC;
uint32_t fReps;
struct timespec tm1;
struct timespec tm2;
};
struct readThr
{
readThr::readThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps)
{
}
void operator()()
{
uint32_t rc = 0;
uint32_t loadedBlocks = 0;
for (uint32_t k = 0; k < fReps; k++)
{
for (uint32_t i = 0; i < fBC.OidRangesList.size(); i++)
{
rc = fBC.ReadOidRanges(fBC.OidRangesList[i]);
cout << ".";
// cout << "-- ReadTest " << OidRangesList[i].oid << " h: " << OidRangesList[i].hwm << "/" << rc <<
// endl;
loadedBlocks += rc;
rc = 0;
}
}
cout << "loadtest " << loadedBlocks << " " << rc << endl << endl;
} // operator()
BCTest& fBC;
uint32_t fReps;
};
void usage()
{
cout << "testbc <cacheSz/1024> <reader threads> <read ahead> <client threads> <reps>" << endl;
}
//
int main(int argc, char* argv[])
{
int cacheSz = 128; // K number of blocks
int thr = 1;
int ra = 1024;
int clients = 1;
int reps = 1;
if (argc > 1 && atoi(argv[1]) > 0)
cacheSz = atoi(argv[1]) * 1024;
if (argc > 2 && atoi(argv[2]) > 0)
thr = atoi(argv[2]);
if (argc > 3 && atoi(argv[3]) > 0)
ra = atoi(argv[3]);
if (argc > 4 && atoi(argv[4]) > 0)
clients = atoi(argv[4]);
if (argc > 5 && atoi(argv[5]) > 0)
reps = atoi(argv[5]);
BCTest bc(cacheSz, thr, ra);
cout << "Cache Size: " << cacheSz << " read Threads: " << thr << " read Ahead: " << ra
<< " clients: " << clients << " repetitions: " << reps << " max Blocks: " << bc.maxBlocksAvailable
<< endl;
// loader test
struct loadThr loader1(bc, reps);
vector<boost::thread*> v;
for (int i = 0; i < clients; i++)
{
boost::thread* th1 = new boost::thread(loader1);
v.push_back(th1);
}
for (int i = 0; i < clients; i++)
{
boost::thread* th1 = v[i];
th1->join();
delete th1;
}
v.clear();
// reader test
return 0;
} // end main

354
utils/testbc/testbc2.cpp
View File

@@ -1,354 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
//
// C++ Implementation: bcTest
//
// Description: A simple Test driver for the Disk Block Buffer Cache
//
// Author: Jason Rodriguez <jrodriguez@calpont.com>, (C) 2007
//
//
#include <vector>
#include <string>
#include <iomanip>
#include <iostream>
#include <sys/time.h>
#include <unistd.h>
#include <boost/thread/thread.hpp>
#include "blockrequestprocessor.h"
#include "blockcacheclient.h"
#include "stats.h"
#include "brm.h"
#include "logger.h"
#include "iomanager.h"
using namespace BRM;
using namespace dbbc;
using namespace std;
using namespace logging;
using namespace primitiveprocessor;
Stats* gPMStatsPtr = NULL;
bool gPMProfOn = false;
uint32_t gSession = 0;
uint32_t lastRangeListIdx = 0;
const uint32_t maxLoadBlocks(1024 * 1024);
void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, double& tm)
{
tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec);
}
namespace primitiveprocessor
{
Logger ml;
}
class BCTest
{
public:
struct OidRanges
{
OID_t oid;
HWM_t hwm;
LBIDRange_v ranges;
OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r)
{
oid = o;
hwm = h;
ranges = r;
}
}; // struct OidRanges
BCTest(const int cacheSz = 64 * 1024, int readThr = 2, int readAhead = 1024);
typedef OidRanges OidRanges_t;
typedef vector<OidRanges_t> OidRangesList_t;
OidRangesList_t OidRangesList;
DBRM dbrm;
uint32_t extentSize;
BRM::OID_t maxOid;
int fCacheSz;
int fReadThr;
int fReadAhead;
uint32_t maxBlocksAvailable;
uint32_t fExtentSize;
void setUp();
int LoadOid(const OidRanges_t& o, uint32_t& loadCount);
void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver);
int ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss);
void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver);
BlockRequestProcessor BRP;
}; // class BCTest
BCTest::BCTest(int cacheSz, int readThr, int readAhead)
: fCacheSz(cacheSz), fReadThr(readThr), fReadAhead(readAhead), BRP(fCacheSz, fReadThr, fReadAhead)
{
setUp();
} // ctor
//
void BCTest::setUp()
{
LBIDRange_v r;
HWM_t hwm;
OID_t oid = 1000;
extentSize = dbrm.getExtentSize();
maxBlocksAvailable = 0;
int i = 0;
fExtentSize = dbrm.getExtentSize();
while (oid < 5000)
{
int ret = 0;
ret = dbrm.lookup(oid, r);
if (ret == 0 && r.size() > 0)
{
dbrm.getHWM(oid, hwm);
maxBlocksAvailable += (r.size() * extentSize);
OidRanges_t oid_range(oid, hwm, r);
OidRangesList.push_back(oid_range);
// cout << "Setup i: " << i++ << " o: " << oid
// << " r: " << ret << " s: " << r.size()
// << " m: " << maxBlocksAvailable
// << endl;
hwm = 0;
r.clear();
}
oid++;
}
// cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl;
i = 0;
} // setUp()
int BCTest::LoadOid(const OidRanges_t& o, uint32_t& loadCount)
{
blockCacheClient bc(BRP);
uint32_t rCount = 0;
for (uint32_t i = 0; i < o.ranges.size(); i++)
{
const InlineLBIDRange r = {o.ranges[i].start, o.ranges[i].size};
if (r.size > 0)
{
bc.check(r, 0, rCount);
// cout << "i: " << i << " c: " << rCount << " " << o.ranges[i].size << endl;
loadCount += rCount;
}
rCount = 0;
} // for
// cout << "hwm: " << o.hwm << " tot: " << loadCount << " " << o.ranges.size() << endl;
return loadCount;
} // LoadOid
int BCTest::ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss)
{
blockCacheClient bc(BRP);
uint8_t inBuff[8192];
int32_t readBlocks = 0;
int32_t missBlocks = 0;
for (uint32_t i = 0; i < v.ranges.size(); i++)
{
FileBuffer fb(-1, -1);
const InlineLBIDRange r = {v.ranges[i].start, v.ranges[i].size};
for (int j = r.start; readBlocks < fCacheSz && j < r.start + r.size; j++)
{
FileBuffer* ptr = bc.getBlockPtr(j, 0);
if (ptr)
{
readBlocks++;
memcpy(inBuff, ptr->getData(), 8192);
}
else
missBlocks++;
}
*hits += readBlocks;
*miss += missBlocks;
missBlocks = 0;
readBlocks = 0;
// cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: "
// << v.hwm << endl;
}
return *hits;
} // ReadRange
// add one block to block cache
//
void BCTest::LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
blockCacheClient bc(BRP);
bool b;
bc.check(lbid, ver, false, b);
} // LoadLbid
// get one block out of block cache
//
void BCTest::ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
uint8_t d[8192] = {'\0'};
blockCacheClient bc(BRP);
bc.read(lbid, ver, d);
} // ReadLbid
struct loadThr
{
loadThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps)
{
}
void operator()()
{
uint32_t loadedBlocks = 0;
uint32_t readBlocks = 0;
uint32_t missBlocks = 0;
uint32_t oidBlocks;
uint32_t i = 0;
uint32_t rc = 0;
clock_gettime(CLOCK_REALTIME, &tm1);
for (uint32_t j = 0; j < fReps; j++)
for (i = 0; loadedBlocks < maxLoadBlocks && i < fBC.OidRangesList.size(); i++)
{
oidBlocks = 0;
rc = fBC.LoadOid(fBC.OidRangesList[i], oidBlocks);
/**
cout << "."
<< "-- " << i << " " << fBC.OidRangesList[i].oid
<< " h: " << fBC.OidRangesList[i].hwm
<< "/" << oidBlocks
<< endl;
**/
loadedBlocks += oidBlocks;
readBlocks += fBC.ReadOidRanges(fBC.OidRangesList[i], &readBlocks, &missBlocks);
}
clock_gettime(CLOCK_REALTIME, &tm2);
double tm3;
timespec_sub(tm1, tm2, tm3);
lastRangeListIdx = i;
cout << "loadtest ld: " << loadedBlocks << " rd: " << readBlocks << "/" << missBlocks << " sz: "
<< fBC.fCacheSz
//<< " last: " << lastRangeListIdx
<< " tm: " << right << setw(10) << fixed << tm3 << endl;
} // operator()
BCTest& fBC;
uint32_t fReps;
struct timespec tm1;
struct timespec tm2;
};
struct readThr
{
readThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps)
{
}
void operator()()
{
for (uint32_t k = 0; k < fReps; k++)
{
}
} // operator()
BCTest& fBC;
uint32_t fReps;
};
void usage()
{
cout << "testbc <cacheSz/1024> <reader threads> <read ahead> <client threads> <reps>" << endl;
}
//
int main(int argc, char* argv[])
{
int cacheSz = 128; // K number of blocks
int thr = 1;
int ra = 1024;
int clients = 1;
int reps = 1;
if (argc > 1 && atoi(argv[1]) > 0)
cacheSz = atoi(argv[1]) * 1024;
if (argc > 2 && atoi(argv[2]) > 0)
thr = atoi(argv[2]);
if (argc > 3 && atoi(argv[3]) > 0)
ra = atoi(argv[3]);
if (argc > 4 && atoi(argv[4]) > 0)
clients = atoi(argv[4]);
if (argc > 5 && atoi(argv[5]) > 0)
reps = atoi(argv[5]);
BCTest bc(cacheSz, thr, ra);
cout << "Cache Size: " << cacheSz << " read Threads: " << thr << " read Ahead: " << ra
<< " clients: " << clients << " repetitions: " << reps << " max Blocks: " << bc.maxBlocksAvailable
<< endl;
// loader test
struct loadThr loader1(bc, reps);
vector<boost::thread*> v;
for (int i = 0; i < clients; i++)
{
boost::thread* th1 = new boost::thread(loader1);
v.push_back(th1);
}
for (int i = 0; i < clients; i++)
{
boost::thread* th1 = v[i];
th1->join();
delete th1;
}
v.clear();
return 0;
} // end main

394
utils/testbc/testbc3.cpp
View File

@@ -1,394 +0,0 @@
/* Copyright (C) 2014 InfiniDB, Inc.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
//
// C++ Implementation: bcTest
//
// Description: A simple Test driver for the Disk Block Buffer Cache
//
// Author: Jason Rodriguez <jrodriguez@calpont.com>, (C) 2007
//
//
#include <vector>
#include <string>
#include <iomanip>
#include <iostream>
#include <sys/time.h>
#include <unistd.h>
#include <boost/thread/thread.hpp>
#include "blockrequestprocessor.h"
#include "blockcacheclient.h"
#include "stats.h"
#include "brm.h"
#include "logger.h"
#include "iomanager.h"
using namespace BRM;
using namespace dbbc;
using namespace std;
using namespace logging;
using namespace primitiveprocessor;
Stats* gPMStatsPtr = NULL;
bool gPMProfOn = false;
uint32_t gSession = 0;
uint32_t lastRangeListIdx = 0;
const uint32_t maxLoadBlocks(1024 * 1024);
void timespec_sub(const struct timespec& tv1, const struct timespec& tv2, double& tm)
{
tm = (double)(tv2.tv_sec - tv1.tv_sec) + 1.e-9 * (tv2.tv_nsec - tv1.tv_nsec);
}
namespace primitiveprocessor
{
Logger ml;
}
class BCTest
{
public:
struct OidRanges
{
OID_t oid;
HWM_t hwm;
LBIDRange_v ranges;
OidRanges(const OID_t o, const HWM_t h, const LBIDRange_v r)
{
oid = o;
hwm = h;
ranges = r;
}
}; // struct OidRanges
BCTest(const int cacheSz = 64 * 1024, int readThr = 2, int readAhead = 1024);
typedef OidRanges OidRanges_t;
typedef vector<OidRanges_t> OidRangesList_t;
OidRangesList_t OidRangesList;
DBRM dbrm;
uint32_t extentSize;
BRM::OID_t maxOid;
int fCacheSz;
int fReadThr;
int fReadAhead;
uint32_t maxBlocksAvailable;
uint32_t fExtentSize;
void setUp();
int LoadOid(const OidRanges_t& o, uint32_t& loadCount);
void LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver);
int ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss);
void ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver);
BlockRequestProcessor BRP;
}; // class BCTest
BCTest::BCTest(int cacheSz, int readThr, int readAhead)
: fCacheSz(cacheSz), fReadThr(readThr), fReadAhead(readAhead), BRP(fCacheSz, fReadThr, fReadAhead)
{
setUp();
} // ctor
//
void BCTest::setUp()
{
LBIDRange_v r;
HWM_t hwm;
OID_t oid = 1000;
extentSize = dbrm.getExtentSize();
maxBlocksAvailable = 0;
int i = 0;
fExtentSize = dbrm.getExtentSize();
while (oid < 5000)
{
int ret = 0;
ret = dbrm.lookup(oid, r);
if (ret == 0 && r.size() > 0)
{
dbrm.getHWM(oid, hwm);
maxBlocksAvailable += (r.size() * extentSize);
OidRanges_t oid_range(oid, hwm, r);
OidRangesList.push_back(oid_range);
// cout << "Setup i: " << i++ << " o: " << oid
// << " r: " << ret << " s: " << r.size()
// << " m: " << maxBlocksAvailable
// << endl;
hwm = 0;
r.clear();
}
oid++;
}
// cout << "\t" << OidRangesList.size() << " oid ranges loaded " << endl << endl;
i = 0;
} // setUp()
int BCTest::LoadOid(const OidRanges_t& o, uint32_t& loadCount)
{
blockCacheClient bc(BRP);
uint32_t rCount = 0;
for (uint32_t i = 0; i < o.ranges.size(); i++)
{
const InlineLBIDRange r = {o.ranges[i].start, o.ranges[i].size};
if (r.size > 0)
{
bc.check(r, 0, rCount);
// cout << "i: " << i << " c: " << rCount << " " << o.ranges[i].size << endl;
loadCount += rCount;
}
rCount = 0;
} // for
// cout << "hwm: " << o.hwm << " tot: " << loadCount << " " << o.ranges.size() << endl;
return loadCount;
} // LoadOid
int BCTest::ReadOidRanges(const OidRanges_t& v, uint32_t* hits, uint32_t* miss)
{
blockCacheClient bc(BRP);
uint32_t readBlocks = 0;
uint32_t missBlocks = 0;
uint8_t inBuff[8192];
int ret;
for (uint32_t i = 0; i < v.ranges.size() && i < lastRangeListIdx; i++)
{
FileBuffer fb(-1, -1);
const InlineLBIDRange r = {v.ranges[i].start, v.ranges[i].size};
for (int j = r.start; j < r.start + r.size; j++)
{
ret = bc.read(j, 0, fb);
FileBuffer* ptr = bc.getBlockPtr(j, 0);
if (ptr)
{
readBlocks++;
memcpy(inBuff, ptr->getData(), 8192);
}
else
missBlocks++;
}
*hits += readBlocks;
*miss += missBlocks;
// cout << " -- Read range idx: " << i << " hits: " << readBlocks << " miss: " << missBlocks << " hwm: "
// << v.hwm << endl;
}
return readBlocks;
} // ReadRange
// add one block to block cache
//
void BCTest::LoadLbid(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
blockCacheClient bc(BRP);
bool b;
bc.check(lbid, ver, false, b);
} // LoadLbid
// get one block out of block cache
//
void BCTest::ReadOidLbids(const BRM::LBID_t lbid, const BRM::VER_t ver)
{
uint8_t d[8192] = {'\0'};
blockCacheClient bc(BRP);
bc.read(lbid, ver, d);
} // ReadLbid
struct loadThr
{
loadThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps)
{
}
void operator()()
{
uint32_t loadedBlocks = 0;
uint32_t oidBlocks;
uint32_t i = 0;
uint32_t rc = 0;
struct timespec tm1;
struct timespec tm2;
tm1.tv_sec = 0;
tm1.tv_nsec = 0;
tm2.tv_sec = 0;
tm2.tv_nsec = 0;
double tm3 = 0;
clock_gettime(CLOCK_REALTIME, &tm1);
for (i = 0; (loadedBlocks + (fBC.OidRangesList[i].ranges.size() * fBC.extentSize)) < fBC.fCacheSz &&
i < fBC.OidRangesList.size();
i++)
{
oidBlocks = 0;
rc = fBC.LoadOid(fBC.OidRangesList[i], oidBlocks);
// cout
// << "-- Load " << i << " " << fBC.OidRangesList[i].oid
// << " h: " << fBC.OidRangesList[i].hwm
// << "/" << oidBlocks << " " << loadedBlocks
// << endl;
loadedBlocks += oidBlocks;
} // for (i...
lastRangeListIdx = i;
clock_gettime(CLOCK_REALTIME, &tm2);
timespec_sub(tm1, tm2, tm3);
cout << "load ld: " << loadedBlocks << " sz: " << fBC.fCacheSz << " rng: " << lastRangeListIdx
<< " tm: " << right << setw(10) << fixed << tm3 << endl;
} // operator()
BCTest& fBC;
uint32_t fReps;
};
struct readThr
{
readThr(BCTest& bc, int reps = 1) : fBC(bc), fReps(reps)
{
}
void operator()()
{
uint32_t rc = 0;
uint32_t readBlocks = 0;
uint32_t hits = 0;
uint32_t miss = 0;
uint32_t hitsTot = 0;
uint32_t missTot = 0;
struct timespec tm1;
struct timespec tm2;
tm1.tv_sec = 0;
tm1.tv_nsec = 0;
tm2.tv_sec = 0;
tm2.tv_nsec = 0;
double tm3 = 0;
clock_gettime(CLOCK_REALTIME, &tm1);
for (uint32_t k = 0; k < fReps; k++)
{
for (uint32_t i = 0; i < lastRangeListIdx && i < fBC.OidRangesList.size(); i++)
{
rc = fBC.ReadOidRanges(fBC.OidRangesList[i], &hits, &miss);
// cout << "-- ReadTest " << fBC.OidRangesList[i].oid << " h: " << fBC.OidRangesList[i].hwm << "/" <<
// rc << endl;
readBlocks += rc;
rc = 0;
}
hitsTot += hits;
missTot += miss;
hits = 0;
miss = 0;
}
clock_gettime(CLOCK_REALTIME, &tm2);
timespec_sub(tm1, tm2, tm3);
cout << "readtest rd: " << hitsTot << "/" << missTot << " sz: " << fBC.fCacheSz << " tm: " << right
<< setw(10) << fixed << tm3 << endl;
} // operator()
BCTest& fBC;
uint32_t fReps;
};
void usage()
{
cout << "testbc <cacheSz/1024> <reader threads> <read ahead> <client threads> <reps>" << endl;
}
//
int main(int argc, char* argv[])
{
int cacheSz = 128; // K number of blocks
int thr = 1;
int ra = 1024;
int clients = 1;
int reps = 1;
if (argc > 1 && atoi(argv[1]) > 0)
cacheSz = atoi(argv[1]) * 1024;
if (argc > 2 && atoi(argv[2]) > 0)
thr = atoi(argv[2]);
if (argc > 3 && atoi(argv[3]) > 0)
ra = atoi(argv[3]);
if (argc > 4 && atoi(argv[4]) > 0)
clients = atoi(argv[4]);
if (argc > 5 && atoi(argv[5]) > 0)
reps = atoi(argv[5]);
BCTest bc(cacheSz, thr, ra);
cout << "Cache Size: " << cacheSz << " read Threads: " << thr << " read Ahead: " << ra
<< " clients: " << clients << " repetitions: " << reps << " max Blocks: " << bc.maxBlocksAvailable
<< endl;
// loader cache
struct loadThr loader1(bc, 1);
struct readThr reader1(bc, reps);
vector<boost::thread*> v;
boost::thread* th1 = new boost::thread(loader1);
th1->join();
for (int i = 0; i < clients; i++)
{
boost::thread* rd1 = new boost::thread(reader1);
v.push_back(rd1);
}
for (int i = 0; i < clients; i++)
{
boost::thread* rd1 = v[i];
rd1->join();
delete rd1;
}
v.clear();
delete th1;
return 0;
} // end main

12
utils/threadpool/CMakeLists.txt
View File

@@ -1,13 +1,7 @@
include_directories(${ENGINE_COMMON_INCLUDES})
# ########## next target ###############
set(threadpool_LIB_SRCS weightedthreadpool.cpp threadpool.cpp prioritythreadpool.cpp fair_threadpool.cpp)
columnstore_library(threadpool ${threadpool_LIB_SRCS})
add_dependencies(threadpool loggingcpp external_boost)
target_link_libraries(threadpool boost_chrono)
install(
TARGETS threadpool
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(threadpool PRIVATE boost_chrono loggingcpp)
add_dependencies(threadpool external_boost)

22
utils/udfsdk/CMakeLists.txt
View File

@@ -10,24 +10,10 @@ set(udfsdk_LIB_SRCS
distinct_count.cpp
)
add_definitions(-DMYSQL_DYNAMIC_PLUGIN)
columnstore_library(udfsdk ${udfsdk_LIB_SRCS})
columnstore_link(udfsdk PRIVATE loggingcpp)
add_dependencies(udfsdk loggingcpp)
install(
TARGETS udfsdk
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
# Do anyone use it?
add_definitions(-DMYSQL_DYNAMIC_PLUGIN)
set(udf_mysql_LIB_SRCS udfmysql.cpp)
add_library(udf_mysql SHARED ${udf_mysql_LIB_SRCS})
install(
TARGETS udf_mysql
DESTINATION ${MARIADB_PLUGINDIR}
COMPONENT columnstore-engine
)
columnstore_mysql_plugin_library(udf_mysql SHARED ${udf_mysql_LIB_SRCS})

9
utils/udfsdk/mcsv1_udaf.h
View File

@@ -129,7 +129,8 @@ class mcsv1Context;
struct UserData
{
UserData() : size(0), data(nullptr){};
UserData() : size(0), data(nullptr) {};
UserData(UserData&) = delete;
explicit UserData(size_t sz)
{
size = sz;
@@ -174,10 +175,6 @@ struct UserData
// The default data store. You may or may not wish to use these fields.
uint32_t size;
uint8_t* data;
private:
// For now, copy construction is unwanted
UserData(UserData&);
};
// Flags to define the type and limitations of a UDA(n)F
@@ -446,7 +443,7 @@ struct ColumnDatum
std::string alias; // Only filled in for init()
uint32_t charsetNumber; // For string collations
ColumnDatum()
: dataType(execplan::CalpontSystemCatalog::UNDEFINED), scale(0), precision(-1), charsetNumber(8){};
: dataType(execplan::CalpontSystemCatalog::UNDEFINED), scale(0), precision(-1), charsetNumber(8) {};
};
// Override mcsv1_UDAF to build your User Defined Aggregate (UDAF) and/or

8
utils/windowfunction/CMakeLists.txt
View File

@@ -25,10 +25,4 @@ set(windowfunction_LIB_SRCS
columnstore_library(windowfunction ${windowfunction_LIB_SRCS})
add_dependencies(windowfunction loggingcpp)
install(
TARGETS windowfunction
DESTINATION ${ENGINE_LIBDIR}
COMPONENT columnstore-engine
)
columnstore_link(windowfunction loggingcpp)