1
0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00

MCOL-4483: Fix and consolidate log files and cpimport logging.

This commit is contained in:
benthompson15
2021-01-20 12:04:52 -06:00
parent ceae2118a2
commit afa88866bb
32 changed files with 172 additions and 287 deletions

View File

@ -40,6 +40,7 @@ SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib)
SET (ENGINE_SYSCONFDIR "/etc") SET (ENGINE_SYSCONFDIR "/etc")
SET (ENGINE_DATADIR "/var/lib/columnstore") SET (ENGINE_DATADIR "/var/lib/columnstore")
SET (ENGINE_LOGDIR "/var/log/mariadb/columnstore")
IF (INSTALL_LAYOUT) IF (INSTALL_LAYOUT)
# We are building from MariaDB server submodule if this is set # We are building from MariaDB server submodule if this is set

View File

@ -36,6 +36,8 @@ using namespace execplan;
#include "elementtype.h" #include "elementtype.h"
using namespace joblist; using namespace joblist;
#include "mcsconfig.h"
boost::mutex fileLock_g; boost::mutex fileLock_g;
namespace namespace
@ -43,7 +45,10 @@ namespace
void logDiskIoInfo(uint64_t stepId, const AnyDataListSPtr& spdl) void logDiskIoInfo(uint64_t stepId, const AnyDataListSPtr& spdl)
{ {
boost::mutex::scoped_lock lk(fileLock_g); boost::mutex::scoped_lock lk(fileLock_g);
ofstream umDiskIoFile("/var/log/mariadb/columnstore/trace/umdiskio.log", ios_base::app); string umDiskioLog = string(MCSLOGDIR) + "/trace/umdiskio.log";
string umDiskioBak = string(MCSLOGDIR) + "/trace/umdiskio.bak";
ofstream umDiskIoFile(umDiskioLog.c_str(), ios_base::app);
CalpontSystemCatalog::OID oid; CalpontSystemCatalog::OID oid;
uint64_t maxBuckets = 0; uint64_t maxBuckets = 0;
@ -107,7 +112,8 @@ void logDiskIoInfo(uint64_t stepId, const AnyDataListSPtr& spdl)
// move the current file to bak when size above .5 G, so total log is 1 G // move the current file to bak when size above .5 G, so total log is 1 G
if (curPos > 0x20000000) if (curPos > 0x20000000)
{ {
(void)system("/bin/mv /var/log/mariadb/columnstore/trace/umdiskio.log /var/log/mariadb/columnstore/trace/umdiskio.bak"); string cmd = "/bin/mv " + umDiskioLog + " " + umDiskioBak;
(void)system(cmd.c_str());
} }
} }

View File

@ -11,7 +11,6 @@ usr/bin/StorageManager
usr/bin/WriteEngineServer usr/bin/WriteEngineServer
usr/bin/clearShm usr/bin/clearShm
usr/bin/cleartablelock usr/bin/cleartablelock
usr/bin/columnstore
usr/bin/columnstore-post-install usr/bin/columnstore-post-install
usr/bin/columnstore-pre-uninstall usr/bin/columnstore-pre-uninstall
usr/bin/columnstoreDBWrite usr/bin/columnstoreDBWrite

View File

@ -389,6 +389,8 @@
#define MCSBINDIR "${ENGINE_BINDIR}" #define MCSBINDIR "${ENGINE_BINDIR}"
#define MCSLOGDIR "${ENGINE_LOGDIR}"
#define MCSMYCNFDIR "${MARIADB_MYCNFDIR}" #define MCSMYCNFDIR "${MARIADB_MYCNFDIR}"
#endif #endif

View File

@ -249,10 +249,6 @@
<ExternalCriticalThreshold>90</ExternalCriticalThreshold> <ExternalCriticalThreshold>90</ExternalCriticalThreshold>
<ExternalMajorThreshold>80</ExternalMajorThreshold> <ExternalMajorThreshold>80</ExternalMajorThreshold>
<ExternalMinorThreshold>70</ExternalMinorThreshold> <ExternalMinorThreshold>70</ExternalMinorThreshold>
<!-- <TempDiskPath>/tmp</TempDiskPath>
<WorkingDir>/tmp</WorkingDir>
<TempFileDir>/tmp/columnstore_tmp_files</TempFileDir>
-->
<TransactionArchivePeriod>10</TransactionArchivePeriod> <TransactionArchivePeriod>10</TransactionArchivePeriod>
<NMSIPAddress>0.0.0.0</NMSIPAddress> <NMSIPAddress>0.0.0.0</NMSIPAddress>
<TempSaveSize>128M</TempSaveSize> <!-- default SWSDL max element save size --> <TempSaveSize>128M</TempSaveSize> <!-- default SWSDL max element save size -->
@ -363,7 +359,7 @@
<FirstOID>3000</FirstOID> <FirstOID>3000</FirstOID>
</OIDManager> </OIDManager>
<WriteEngine> <WriteEngine>
<BulkRoot>/var/lib/columnstore/data/bulk</BulkRoot> <BulkRoot>/var/log/mariadb/columnstore/data/bulk</BulkRoot>
<BulkRollbackDir>/var/lib/columnstore/data1/systemFiles/bulkRollback</BulkRollbackDir> <BulkRollbackDir>/var/lib/columnstore/data1/systemFiles/bulkRollback</BulkRollbackDir>
<MaxFileSystemDiskUsagePct>98</MaxFileSystemDiskUsagePct> <MaxFileSystemDiskUsagePct>98</MaxFileSystemDiskUsagePct>
<CompressedPaddingBlocks>1</CompressedPaddingBlocks> <!-- Number of blocks used to pad compressed chunks --> <CompressedPaddingBlocks>1</CompressedPaddingBlocks> <!-- Number of blocks used to pad compressed chunks -->

View File

@ -3,7 +3,6 @@ SET (DEFAULT_GROUP "mysql")
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslogSetup.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslogSetup.sh" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslogSetup.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslogSetup.sh" @ONLY)
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstore-post-install.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstore-post-install" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstore-post-install.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstore-post-install" @ONLY)
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstore.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstore" @ONLY)
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstore-pre-uninstall.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstore-pre-uninstall" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstore-pre-uninstall.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstore-pre-uninstall" @ONLY)
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreLogRotate.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreLogRotate" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreLogRotate.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreLogRotate" @ONLY)
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mariadb-columnstore.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mariadb-columnstore.service" @ONLY) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mariadb-columnstore.service.in" "${CMAKE_CURRENT_SOURCE_DIR}/mariadb-columnstore.service" @ONLY)
@ -27,7 +26,6 @@ install(PROGRAMS columnstore-post-install
columnstore_run.sh columnstore_run.sh
post-mysql-install post-mysql-install
post-mysqld-install post-mysqld-install
columnstore
columnstoreSyslogSetup.sh columnstoreSyslogSetup.sh
mcs-stop-controllernode.sh mcs-stop-controllernode.sh
mcs-loadbrm.py mcs-loadbrm.py

View File

@ -32,22 +32,22 @@ fi
checkForError() { checkForError() {
# check for password error # check for password error
grep "ERROR 1045" ${tmpDir}/mysql_install.log > ${tmpDir}/error.check grep "ERROR 1045" ${installLogDir}/mysql_install.log > ${installLogDir}/error.check
if [ `cat ${tmpDir}/error.check | wc -c` -ne 0 ]; then if [ `cat ${installLogDir}/error.check | wc -c` -ne 0 ]; then
echo "There were authentication issues running install_mcs_mysql.sh \ echo "There were authentication issues running install_mcs_mysql.sh \
script. The log is available in ${tmpDir}/mysql_install.log. Please re-run \ script. The log is available in ${installLogDir}/mysql_install.log. Please re-run \
columnstore-post-install manually after solving the authentication issues." columnstore-post-install manually after solving the authentication issues."
rm -f ${tmpDir}/error.check rm -f ${installLogDir}/error.check
return 2; return 2;
fi fi
rm -f ${tmpDir}/error.check rm -f ${installLogDir}/error.check
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
# See if engine columnstore exist # See if engine columnstore exist
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
echo "checking for engine columnstore..." echo "checking for engine columnstore..."
$MDB --execute="show engines" 2> ${tmpDir}/post-mysql-install.log | grep -i columnstore >> ${tmpDir}/post-mysql-install.log 2>&1 $MDB --execute="show engines" 2> ${installLogDir}/post-mysql-install.log | grep -i columnstore >> ${installLogDir}/post-mysql-install.log 2>&1
# #
# Add compressiontype column to SYSCOLUMN if applicable # Add compressiontype column to SYSCOLUMN if applicable
@ -103,35 +103,50 @@ profileFile="/etc/profile.d/columnstoreAlias.sh"
/bin/cp -f @ENGINE_SUPPORTDIR@/columnstoreAlias /etc/profile.d/columnstoreAlias.sh /bin/cp -f @ENGINE_SUPPORTDIR@/columnstoreAlias /etc/profile.d/columnstoreAlias.sh
chmod 644 /etc/profile.d/columnstoreAlias.sh >/dev/null 2>&1 chmod 644 /etc/profile.d/columnstoreAlias.sh >/dev/null 2>&1
test -d /var/log/mariadb/columnstore || mkdir -p /var/log/mariadb/columnstore >/dev/null 2>&1 test -d @ENGINE_LOGDIR@ || mkdir -p @ENGINE_LOGDIR@ >/dev/null 2>&1
test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 test -d @ENGINE_LOGDIR@/archive || mkdir @ENGINE_LOGDIR@/archive >/dev/null 2>&1
test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 test -d @ENGINE_LOGDIR@/corefiles || mkdir @ENGINE_LOGDIR@/corefiles >/dev/null 2>&1
test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 test -d @ENGINE_LOGDIR@/trace || mkdir @ENGINE_LOGDIR@/trace >/dev/null 2>&1
test -h /var/log/mariadb/columnstore/data && rm -f /var/log/mariadb/columnstore/data test -d @ENGINE_LOGDIR@/cpimport || mkdir @ENGINE_LOGDIR@/cpimport >/dev/null 2>&1
chmod 755 /var/log/mariadb/columnstore/corefiles > /dev/null 2>&1 test -d @ENGINE_LOGDIR@/install || mkdir @ENGINE_LOGDIR@/install >/dev/null 2>&1
test -h @ENGINE_LOGDIR@/data && rm -f @ENGINE_LOGDIR@/data
chmod 755 @ENGINE_LOGDIR@/corefiles > /dev/null 2>&1
chmod 777 @ENGINE_LOGDIR@/cpimport
chmod 777 @ENGINE_LOGDIR@/install
installLogDir=@ENGINE_LOGDIR@/install
# make sure trace dir is world-writable and sticky # make sure trace dir is world-writable and sticky
test -d /var/lib/columnstore/data || mkdir -p /var/lib/columnstore/data test -d @ENGINE_DATADIR@/data || mkdir -p @ENGINE_DATADIR@/data
test -d /var/lib/columnstore/data1/systemFiles/dbrm || mkdir -p /var/lib/columnstore/data1/systemFiles/dbrm test -d @ENGINE_DATADIR@/data1/systemFiles/dbrm || mkdir -p @ENGINE_DATADIR@/data1/systemFiles/dbrm
test -d /var/lib/columnstore/local || mkdir -p /var/lib/columnstore/local test -d @ENGINE_DATADIR@/local || mkdir -p @ENGINE_DATADIR@/local
test -d /var/lib/columnstore/data1/systemFiles/dataTransaction || rmdir /var/lib/columnstore/data1/systemFiles/dataTransaction >/dev/null 2>&1 test -d @ENGINE_DATADIR@/data1/systemFiles/dataTransaction || rmdir @ENGINE_DATADIR@/data1/systemFiles/dataTransaction >/dev/null 2>&1
test -d /var/lib/columnstore/data1/systemFiles/dataTransaction/archive || rmdir /var/lib/columnstore/data1/systemFiles/dataTransaction/archive >/dev/null 2>&1 test -d @ENGINE_DATADIR@/data1/systemFiles/dataTransaction/archive || rmdir @ENGINE_DATADIR@/data1/systemFiles/dataTransaction/archive >/dev/null 2>&1
chmod 1755 /var/lib/columnstore/data1 >/dev/null 2>&1 chmod 1755 @ENGINE_DATADIR@/data1 >/dev/null 2>&1
chmod -R 1755 /var/lib/columnstore/data1/systemFiles >/dev/null 2>&1 chmod -R 1755 @ENGINE_DATADIR@/data1/systemFiles >/dev/null 2>&1
chmod 1755 @ENGINE_SYSCONFDIR@/columnstore > /dev/null 2>&1 chmod 1755 @ENGINE_SYSCONFDIR@/columnstore > /dev/null 2>&1
#create the bulk-load dirs #create the bulk-load dirs
mkdir -p /var/lib/columnstore/data/bulk/data/import >/dev/null 2>&1 mkdir -p @ENGINE_DATADIR@/data/bulk/data/import >/dev/null 2>&1
mkdir -p /var/lib/columnstore/data/bulk/log >/dev/null 2>&1 mkdir -p @ENGINE_DATADIR@/data/bulk/log >/dev/null 2>&1
mkdir -p /var/lib/columnstore/data/bulk/job >/dev/null 2>&1 mkdir -p @ENGINE_DATADIR@/data/bulk/job >/dev/null 2>&1
mkdir -p /var/lib/columnstore/data/bulk/rollback >/dev/null 2>&1 mkdir -p @ENGINE_DATADIR@/data/bulk/rollback >/dev/null 2>&1
mkdir -p /var/lib/columnstore/data/bulk/tmpjob >/dev/null 2>&1 mkdir -p @ENGINE_DATADIR@/data/bulk/tmpjob >/dev/null 2>&1
rm -f /var/lib/columnstore/data/bulk/tmpjob/* >/dev/null 2>&1 mkdir -p @ENGINE_LOGDIR@/data/bulk/data/import >/dev/null 2>&1
mkdir -p @ENGINE_LOGDIR@/data/bulk/log >/dev/null 2>&1
mkdir -p @ENGINE_LOGDIR@/data/bulk/job >/dev/null 2>&1
mkdir -p @ENGINE_LOGDIR@/data/bulk/rollback >/dev/null 2>&1
mkdir -p @ENGINE_LOGDIR@/data/bulk/tmpjob >/dev/null 2>&1
rm -f @ENGINE_DATADIR@/data/bulk/tmpjob/* >/dev/null 2>&1
rm -f @ENGINE_LOGDIR@/data/bulk/tmpjob/* >/dev/null 2>&1
chmod -R 777 @ENGINE_LOGDIR@/data
#get columnstore temp file directory name #get columnstore temp file directory name
tmpDir=/tmp/columnstore_tmp_files tmpDir=`@ENGINE_BINDIR@/mcsGetConfig SystemConfig SystemTempFileDir`
scratchDir=$tmpDir`@ENGINE_BINDIR@/mcsGetConfig SystemConfig hdfsRdwrScratch`
mkdir $tmpDir >/dev/null 2>&1 mkdir $tmpDir >/dev/null 2>&1
mkdir $scratchDir >/dev/null 2>&1
chmod 777 $tmpDir chmod 777 $tmpDir
chmod 777 $scratchDir
#create mount directories #create mount directories
mkdir /mnt/tmp > /dev/null 2>&1 mkdir /mnt/tmp > /dev/null 2>&1
@ -196,12 +211,12 @@ if [ $user = "root" ]; then
fi fi
if [ $user = "root" ]; then if [ $user = "root" ]; then
columnstoreSyslogSetup.sh install > $tmpDir/syslog_install.log 2>&1 columnstoreSyslogSetup.sh install > $installLogDir/syslog_install.log 2>&1
#check if MariaDB Columnstore system logging was setup #check if MariaDB Columnstore system logging was setup
cat $tmpDir/syslog_install.log | grep 'No System Logging' >/dev/null 2>&1 cat $installLogDir/syslog_install.log | grep 'No System Logging' >/dev/null 2>&1
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
cat $tmpDir/syslog_install.log cat $installLogDir/syslog_install.log
fi fi
else else
chown $user:$user @ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml chown $user:$user @ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml
@ -231,7 +246,7 @@ rm -f $lockdir/columnstore
# This was the last place of postConfigure. RIP # This was the last place of postConfigure. RIP
@ENGINE_SBINDIR@/install_mcs_mysql.sh --tmpdir=$tmpDir @ENGINE_SBINDIR@/install_mcs_mysql.sh --tmpdir=$installLogDir
# Restart MDB to enable plugin # Restart MDB to enable plugin
systemctl cat mariadb.service > /dev/null 2>&1 systemctl cat mariadb.service > /dev/null 2>&1
@ -326,23 +341,22 @@ fi
#change ownership/permissions to be able to run columnstore as non-root #change ownership/permissions to be able to run columnstore as non-root
# TODO: Remove conditional once container dispatcher uses non-root by default # TODO: Remove conditional once container dispatcher uses non-root by default
if [ $(running_systemd) -eq 0 ]; then if [ $(running_systemd) -eq 0 ]; then
chown -R @DEFAULT_USER@:@DEFAULT_GROUP@ /var/log/mariadb/columnstore chown -R @DEFAULT_USER@:@DEFAULT_GROUP@ @ENGINE_LOGDIR@
chown -R @DEFAULT_USER@:@DEFAULT_GROUP@ /etc/columnstore chown -R @DEFAULT_USER@:@DEFAULT_GROUP@ /etc/columnstore
chown -R @DEFAULT_USER@:@DEFAULT_GROUP@ /var/lib/columnstore chown -R @DEFAULT_USER@:@DEFAULT_GROUP@ @ENGINE_DATADIR@
chown -R @DEFAULT_USER@:@DEFAULT_GROUP@ /tmp/columnstore_tmp_files chown -R @DEFAULT_USER@:@DEFAULT_GROUP@ $tmpDir
chmod 777 /tmp/columnstore_tmp_files
chmod 777 /dev/shm chmod 777 /dev/shm
fi fi
systemctl cat mariadb-columnstore.service > /dev/null 2>&1 systemctl cat mariadb-columnstore.service > /dev/null 2>&1
if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then
mkdir -p /var/lib/columnstore/storagemanager mkdir -p @ENGINE_DATADIR@/storagemanager
chown -R mysql:mysql /var/lib/columnstore/storagemanager chown -R mysql:mysql @ENGINE_DATADIR@/storagemanager
IFLAG=/var/lib/columnstore/storagemanager/storagemanager-lock IFLAG=@ENGINE_DATADIR@/storagemanager/storagemanager-lock
# shared dbroot1 synchronization # shared dbroot1 synchronization
# prevents dbbuilder from processing simultaneously on two or more nodes # prevents dbbuilder from processing simultaneously on two or more nodes
exec {fd_lock}>/var/lib/columnstore/data1/dbroot1-lock exec {fd_lock}>@ENGINE_DATADIR@/data1/dbroot1-lock
flock -x "$fd_lock" flock -x "$fd_lock"
# shared storagemanager data corruption prevention # shared storagemanager data corruption prevention

View File

@ -49,8 +49,7 @@ if [ -n "$(pgrep -x ProcMon)" ] || [ -n "$(pgrep -x ProcMgr)" ];then
fi fi
rm -f /etc/profile.d/columnstoreAlias.sh rm -f /etc/profile.d/columnstoreAlias.sh
rm -f /var/log/mariadb/columnstore/activeAlarms > /dev/null 2>&1 rm -f @ENGINE_LOGDIR@/*.log1 > /dev/null 2>&1
rm -f /var/log/mariadb/columnstore/*.log1 > /dev/null 2>&1
#get temp directory #get temp directory
tmpDir=`@ENGINE_BINDIR@/mcsGetConfig SystemConfig SystemTempFileDir` tmpDir=`@ENGINE_BINDIR@/mcsGetConfig SystemConfig SystemTempFileDir`
@ -59,11 +58,11 @@ if [[ $tmpDir = *"tmp" ]]; then
rm -rf $tmpDir/* rm -rf $tmpDir/*
fi fi
rm -f /var/lib/columnstore/local/*.columnstore rm -f @ENGINE_DATADIR@/local/*.columnstore
rm -rf /var/lib/columnstore/local/etc/ rm -rf @ENGINE_DATADIR@/local/etc/
rm -f /var/lib/columnstore/data/bulk/tmp/job/* >/dev/null 2>&1 rm -f @ENGINE_DATADIR@/data/bulk/tmp/job/* >/dev/null 2>&1
rm -f /var/lib/columnstore/local/moveDbrootTransactionLog rm -f @ENGINE_DATADIR@/local/moveDbrootTransactionLog
rm -rf /var/log/mariadb/columnstore/ rm -rf @ENGINE_LOGDIR@
rm -rf /tmp/columnstore_tmp_files rm -rf /tmp/columnstore_tmp_files
lockdir=`@ENGINE_BINDIR@/mcsGetConfig Installation LockFileDirectory` lockdir=`@ENGINE_BINDIR@/mcsGetConfig Installation LockFileDirectory`
@ -111,7 +110,7 @@ if [ -n "$systemctl" ] && [ $(running_systemd) -eq 0 ]; then
systemctl daemon-reload systemctl daemon-reload
# remove flag to prevent clusters using shared storage from initializing columnstore more than once # remove flag to prevent clusters using shared storage from initializing columnstore more than once
IFLAG=/var/lib/columnstore/storagemanager/storagemanager-lock IFLAG=@ENGINE_DATADIR@/storagemanager/storagemanager-lock
if [ -e $IFLAG ]; then if [ -e $IFLAG ]; then
rm $IFLAG rm $IFLAG
fi fi
@ -137,7 +136,7 @@ fi
rm -f @ENGINE_SYSCONFDIR@/columnstore/AlarmConfig.xml.installSave rm -f @ENGINE_SYSCONFDIR@/columnstore/AlarmConfig.xml.installSave
#remove OAMdbrootCheck file #remove OAMdbrootCheck file
rm -f /var/lib/columnstore/data*/OAMdbrootCheck > /dev/null 2>&1 rm -f @ENGINE_DATADIR@/data*/OAMdbrootCheck > /dev/null 2>&1
rm -f /etc/rsyslog.d/49-columnstore.conf > /dev/null 2>&1 rm -f /etc/rsyslog.d/49-columnstore.conf > /dev/null 2>&1
rm -rf /tmp/columnstore_tmp_files > /dev/null 2>&1 rm -rf /tmp/columnstore_tmp_files > /dev/null 2>&1

View File

@ -1,161 +0,0 @@
#!/bin/bash
#
# $Id: columnstore 3704 2013-08-07 03:33:20Z bwilkinson $
#
# columnstore Starts MariaDB Columnstore database platform
#
#
# chkconfig: 2345 99 99
# description: MariaDB Columnstore is a database platform that utilizes Mysql
#
### BEGIN INIT INFO
# Provides: columnstore
# Required-Start: $local_fs $remote_fs $network $syslog $all
# Required-Stop: $local_fs $remote_fs $network $syslog
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: Start/stop MariaDB Columnstore DW DBMS
### END INIT INFO
running_systemd() {
if [ "$(ps --no-headers -o comm 1)" = "systemd" ]; then
echo 0
else
echo 1
fi
}
USER=`whoami 2>/dev/null`
# Source function library.
if [ -f /etc/init.d/functions ]; then
. /etc/init.d/functions
fi
. @ENGINE_SUPPORTDIR@/columnstore_functions
mt=`module_type`
mid=`module_id`
has_um=`@ENGINE_BINDIR@/mcsGetConfig SystemModuleConfig ModuleCount2`
if [ "x$has_um" = x ]; then
has_um=0
fi
lockdir=`@ENGINE_BINDIR@/mcsGetConfig Installation LockFileDirectory`
#get temp directory
tmpDir=`@ENGINE_BINDIR@/mcsGetConfig SystemConfig SystemTempFileDir`
mkdir $tmpDir >/dev/null 2>&1
if [ $USER == "root" ]; then
chmod 777 $tmpDir
fi
checkInstallSetup() {
InitialInstallFlag=`@ENGINE_BINDIR@/mcsGetConfig -c @ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml Installation InitialInstallFlag`
if [ $InitialInstallFlag != "y" ]; then
echo "Please run the postConfigure install script, check the Installation Guide"
echo "for additional details"
rm -f $lockdir/columnstore
exit 1
fi
}
start() {
if [ -f $lockdir/columnstore ]; then
echo "MariaDB Columnstore Database Platform already running"
exit 0
fi
# Make sure ProcMon is down first
touch ${tmpDir}/StopColumnstore
pkill ProcMon
pkill ProcMgr
(mkdir -p $lockdir && touch $lockdir/columnstore) >/dev/null 2>&1
#checkInstallSetup
CoreFileFlag=`@ENGINE_BINDIR@/mcsGetConfig -c @ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml Installation CoreFileFlag`
if [ $CoreFileFlag = "y" ]; then
#columnstore core files
ulimit -c unlimited > /dev/null 2>&1
sysctl -q -w kernel.core_uses_pid=1 > /dev/null 2>&1
sysctl -q -w kernel.core_pattern=/var/log/mariadb/columnstore/corefiles/core.%e.%p > /dev/null 2>&1
fi
RETVAL=0
moduleName="$(cat /var/lib/columnstore/local/module)"
echo "Starting module $moduleName of MariaDB Columnstore Database Platform"
rm -f ${tmpDir}/StopColumnstore
exec columnstore_run.sh -l ${tmpDir} ProcMon > /dev/null 2>&1 &
return $RETVAL
}
stop() {
moduleName="$(cat /var/lib/columnstore/local/module)"
echo "Stopping module $moduleName of MariaDB Columnstore Database Platform"
touch ${tmpDir}/StopColumnstore
mcsadmin stopModule $moduleName y
RETVAL=$?
rm -f $lockdir/columnstore
fuser -k 8604/tcp > /dev/null 2>&1
# Test we are using systemd
systemctl cat mariadb.service > /dev/null 2>&1
if [ $? -eq 0 ] && [ $(running_systemd) -eq 0 ]; then
systemctl stop mariadb.service > /dev/null 2>&1
else
pkill mysqld
fi
pkill ProcMon
pkill ProcMgr
return $RETVAL
}
restart() {
stop
start
}
status() {
isrunning=0
if [ $EUID -eq 0 ]; then
if [ -f $lockdir/columnstore ]; then
isrunning=1
fi
else
pgrep ProcMon >/dev/null 2>&1
if [ $? -eq 0 ]; then
isrunning=1
fi
fi
if [ $isrunning -ne 0 ]; then
echo "MariaDB Columnstore is running"
else
echo "MariaDB Columnstore is not running"
exit 3
fi
}
case "$1" in
start)
start
;;
stop)
stop
;;
restart)
restart
;;
condrestart)
[ -f $lockdir/columnstore ] && restart || :
;;
status)
status
;;
*)
echo $"Usage: $0 {start|stop|restart|condrestart|status}"
exit 1
esac
exit $?

View File

@ -3,13 +3,13 @@
# as part of the package installation # as part of the package installation
# #
/var/log/mariadb/columnstore/*.log { @ENGINE_LOGDIR@/*.log {
missingok missingok
rotate 7 rotate 7
daily daily
dateext dateext
copytruncate copytruncate
olddir /var/log/mariadb/columnstore/archive olddir @ENGINE_LOGDIR@/archive
su root root su root root
} }
@ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml { @ENGINE_SYSCONFDIR@/columnstore/Columnstore.xml {

View File

@ -1,8 +1,8 @@
# MariaDB Columnstore Database Platform Logging # MariaDB Columnstore Database Platform Logging
$FileOwner @DEFAULT_USER@ $FileOwner @DEFAULT_USER@
$FileGroup @DEFAULT_GROUP@ $FileGroup @DEFAULT_GROUP@
local1.=crit -/var/log/mariadb/columnstore/crit.log local1.=crit -@ENGINE_LOGDIR@/crit.log
local1.=err -/var/log/mariadb/columnstore/err.log local1.=err -@ENGINE_LOGDIR@/err.log
local1.=warning -/var/log/mariadb/columnstore/warning.log local1.=warning -@ENGINE_LOGDIR@/warning.log
local1.=info -/var/log/mariadb/columnstore/info.log local1.=info -@ENGINE_LOGDIR@/info.log
local1.=debug -/var/log/mariadb/columnstore/debug.log local1.=debug -@ENGINE_LOGDIR@/debug.log

View File

@ -50,7 +50,7 @@ columnstoreSyslogFile=@ENGINE_SUPPORTDIR@/columnstoreSyslog
columnstoreSyslogFile7=@ENGINE_SUPPORTDIR@/columnstoreSyslog7 columnstoreSyslogFile7=@ENGINE_SUPPORTDIR@/columnstoreSyslog7
#get temp directory #get temp directory
tmpDir=`@ENGINE_BINDIR@/mcsGetConfig SystemConfig SystemTempFileDir` tmpDir=@ENGINE_LOGDIR@/install
checkSyslog() { checkSyslog() {
#check which syslog daemon is being used #check which syslog daemon is being used
@ -159,16 +159,16 @@ fi
} }
makeDir() { makeDir() {
if [ ! -d /var/log/mariadb/columnstore ]; then if [ ! -d @ENGINE_LOGDIR@ ]; then
mkdir -p /var/log/mariadb/columnstore >/dev/null 2>&1 mkdir -p @ENGINE_LOGDIR@ >/dev/null 2>&1
test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 test -d @ENGINE_LOGDIR@/archive || mkdir @ENGINE_LOGDIR@/archive >/dev/null 2>&1
test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 test -d @ENGINE_LOGDIR@/corefiles || mkdir @ENGINE_LOGDIR@/corefiles >/dev/null 2>&1
test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 test -d @ENGINE_LOGDIR@/trace || mkdir @ENGINE_LOGDIR@/trace >/dev/null 2>&1
chown $username:$groupname -R /var/log/mariadb chown $username:$groupname -R /var/log/mariadb
else else
test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1 test -d @ENGINE_LOGDIR@/archive || mkdir @ENGINE_LOGDIR@/archive >/dev/null 2>&1
test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1 test -d @ENGINE_LOGDIR@/corefiles || mkdir @ENGINE_LOGDIR@/corefiles >/dev/null 2>&1
test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1 test -d @ENGINE_LOGDIR@/trace || mkdir @ENGINE_LOGDIR@/trace >/dev/null 2>&1
fi fi
} }
@ -220,19 +220,18 @@ if [ ! -z "$syslog_conf" ] ; then
fi fi
#log install message and find the least permission that allows logging to work #log install message and find the least permission that allows logging to work
CHMOD_LIST=("750" "770" "775" "777") CHMOD_LIST=("755" "775" "777")
for CHMOD in "${CHMOD_LIST[@]}"; do for CHMOD in "${CHMOD_LIST[@]}"; do
chmod $CHMOD /var/log/mariadb chmod $CHMOD /var/log/mariadb
chmod $CHMOD /var/log/mariadb/columnstore chmod $CHMOD @ENGINE_LOGDIR@
chmod $CHMOD /var/log/mariadb/columnstore/archive chmod $CHMOD @ENGINE_LOGDIR@/archive
chmod $CHMOD /var/log/mariadb/columnstore/corefiles chmod $CHMOD @ENGINE_LOGDIR@/corefiles
chmod $CHMOD /var/log/mariadb/columnstore/trace chmod $CHMOD @ENGINE_LOGDIR@/trace
. columnstore_functions cplogger -i 19 "***** MariaDB Columnstore Installed *****"
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib cplogger -i 19 "***** MariaDB Columnstore Installed *****"
if [ -f /var/log/mariadb/columnstore/info.log ]; then if [ -f @ENGINE_LOGDIR@/info.log ]; then
if [ -s /var/log/mariadb/columnstore/info.log ]; then if [ -s @ENGINE_LOGDIR@/info.log ]; then
exit 0 exit 0
fi fi
fi fi
@ -290,7 +289,7 @@ check() {
number=$RANDOM number=$RANDOM
cplogger -i 104 "MariaDB Columnstore Log Test: $number" cplogger -i 104 "MariaDB Columnstore Log Test: $number"
sleep 3 sleep 3
egrep -qs "MariaDB Columnstore Log Test: $number" /var/log/mariadb/columnstore/info.log egrep -qs "MariaDB Columnstore Log Test: $number" @ENGINE_LOGDIR@/info.log
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
echo "MariaDB Columnstore System Logging working" echo "MariaDB Columnstore System Logging working"
exit 0 exit 0

View File

@ -4,7 +4,7 @@
# prevent nodes using shared storage manager from stepping on each other when initializing # prevent nodes using shared storage manager from stepping on each other when initializing
# flock will open up an exclusive file lock to run atomic operations # flock will open up an exclusive file lock to run atomic operations
exec {fd_lock}>/var/lib/columnstore/storagemanager/storagemanager-lock exec {fd_lock}>@ENGINE_DATADIR@/storagemanager/storagemanager-lock
flock -n "$fd_lock" || exit 0 flock -n "$fd_lock" || exit 0
# pass in arg of 1 to start DBRM_Worker1 # pass in arg of 1 to start DBRM_Worker1
@ -15,7 +15,7 @@ flock -n "$fd_lock" || exit 0
/bin/systemctl start mcs-exemgr /bin/systemctl start mcs-exemgr
/bin/systemctl start mcs-dmlproc /bin/systemctl start mcs-dmlproc
/bin/systemctl start mcs-ddlproc /bin/systemctl start mcs-ddlproc
su -s /bin/sh -c '@ENGINE_BINDIR@/dbbuilder 7' @DEFAULT_USER@ 1> /tmp/columnstore_tmp_files/dbbuilder.log su -s /bin/sh -c '@ENGINE_BINDIR@/dbbuilder 7' @DEFAULT_USER@ 1> @ENGINE_LOGDIR@/install/dbbuilder.log
flock -u "$fd_lock" flock -u "$fd_lock"

View File

@ -41,6 +41,7 @@ namespace fs = boost::filesystem;
#include "liboamcpp.h" #include "liboamcpp.h"
#include "messagelog.h" #include "messagelog.h"
#include "messageids.h" #include "messageids.h"
#include "mcsconfig.h"
// Testing definitions. These should never be commented out in checked in code. // Testing definitions. These should never be commented out in checked in code.
#define OAM_AVAILABLE // Comment out to skip the OAM calls to get log file locations and turn on/off logging. #define OAM_AVAILABLE // Comment out to skip the OAM calls to get log file locations and turn on/off logging.
@ -386,7 +387,7 @@ list<string> ReplayTxnLog::getLogFileNames()
oam.getLogFile("pm1", "data", fullLogFileName); oam.getLogFile("pm1", "data", fullLogFileName);
#else #else
*/ */
fullLogFileName = "/var/log/mariadb/columnstore/data/data_mods.log"; fullLogFileName = string(MCSLOGDIR) + "/data/data_mods.log";
/* /*
#endif #endif
*/ */

View File

@ -41,6 +41,7 @@
#include "liboamcpp.h" #include "liboamcpp.h"
#include "configcpp.h" #include "configcpp.h"
#include "installdir.h" #include "installdir.h"
#include "mcsconfig.h"
using namespace std; using namespace std;
using namespace oam; using namespace oam;
@ -319,7 +320,7 @@ int main(int argc, char** argv)
{ {
dbrm.setSystemSuspended(true); dbrm.setSystemSuspended(true);
sleep(5); sleep(5);
string cmd = "save_brm > /var/log/mariadb/columnstore/save_brm.log1 2>&1"; string cmd = "save_brm > " + string(MCSLOGDIR) + "/save_brm.log1 2>&1";
int rtnCode = system(cmd.c_str()); int rtnCode = system(cmd.c_str());
if (rtnCode == 0) if (rtnCode == 0)

View File

@ -34,6 +34,7 @@ using namespace std;
#include "rwlock_local.h" #include "rwlock_local.h"
#include "dbrm.h" #include "dbrm.h"
#include "pp_logger.h" #include "pp_logger.h"
#include "mcsconfig.h"
namespace dbbc namespace dbbc
{ {
@ -69,7 +70,7 @@ BlockRequestProcessor::BlockRequestProcessor(uint32_t numBlcks,
#ifdef _MSC_VER #ifdef _MSC_VER
brpLogFileName << "C:/Calpont/log/trace/brp." << tid; brpLogFileName << "C:/Calpont/log/trace/brp." << tid;
#else #else
brpLogFileName << "/var/log/mariadb/columnstore/trace/brp." << tid; brpLogFileName << MCSLOGDIR << "/trace/brp." << tid;
#endif #endif
fLogFile.open(brpLogFileName.str().c_str(), ios_base::app | ios_base::ate); fLogFile.open(brpLogFileName.str().c_str(), ios_base::app | ios_base::ate);
} }

View File

@ -41,6 +41,7 @@
#include "stats.h" #include "stats.h"
#include "configcpp.h" #include "configcpp.h"
#include "filebuffermgr.h" #include "filebuffermgr.h"
#include "mcsconfig.h"
using namespace config; using namespace config;
using namespace boost; using namespace boost;
@ -73,7 +74,7 @@ FileBufferMgr::FileBufferMgr(const uint32_t numBlcks, const uint32_t blkSz, cons
#ifdef _MSC_VER #ifdef _MSC_VER
fLog.open("C:/Calpont/log/trace/bc", ios_base::app | ios_base::ate); fLog.open("C:/Calpont/log/trace/bc", ios_base::app | ios_base::ate);
#else #else
fLog.open("/var/log/mariadb/columnstore/trace/bc", ios_base::app | ios_base::ate); fLog.open(string(MCSLOGDIR) + "/trace/bc", ios_base::app | ios_base::ate);
#endif #endif
} }

View File

@ -103,6 +103,8 @@ using namespace compress;
#include "IDBLogger.h" #include "IDBLogger.h"
using namespace idbdatafile; using namespace idbdatafile;
#include "mcsconfig.h"
typedef tr1::unordered_set<BRM::OID_t> USOID; typedef tr1::unordered_set<BRM::OID_t> USOID;
namespace primitiveprocessor namespace primitiveprocessor
@ -454,7 +456,7 @@ void* thr_popper(ioManager* arg)
iomLogFileName << "C:/Calpont/log/trace/iom." << threadId; iomLogFileName << "C:/Calpont/log/trace/iom." << threadId;
#else #else
threadId = pthread_self(); threadId = pthread_self();
iomLogFileName << "/var/log/mariadb/columnstore/trace/iom." << threadId; iomLogFileName << MCSLOGDIR << "/trace/iom." << threadId;
#endif #endif
lFile.open(iomLogFileName.str().c_str(), ios_base::app | ios_base::ate); lFile.open(iomLogFileName.str().c_str(), ios_base::app | ios_base::ate);
} }
@ -1351,7 +1353,7 @@ ioManager::ioManager(FileBufferMgr& fbm,
#ifdef _MSC_VER #ifdef _MSC_VER
FDTraceFile().open("C:/Calpont/log/trace/fdcache", ios_base::ate | ios_base::app); FDTraceFile().open("C:/Calpont/log/trace/fdcache", ios_base::ate | ios_base::app);
#else #else
FDTraceFile().open("/var/log/mariadb/columnstore/trace/fdcache", ios_base::ate | ios_base::app); FDTraceFile().open(string(MCSLOGDIR) + "/trace/fdcache", ios_base::ate | ios_base::app);
#endif #endif
} }

View File

@ -42,6 +42,7 @@ using namespace boost;
#include "stats.h" #include "stats.h"
#include "messagelog.h" #include "messagelog.h"
#include "exceptclasses.h" #include "exceptclasses.h"
#include "mcsconfig.h"
using namespace BRM; using namespace BRM;
@ -110,7 +111,7 @@ public:
#ifdef _MSC_VER #ifdef _MSC_VER
oss << "C:/Calpont/log/trace/" << outName << '.' << sessionID; oss << "C:/Calpont/log/trace/" << outName << '.' << sessionID;
#else #else
oss << "/var/log/mariadb/columnstore/trace/" << outName << '.' << sessionID; oss << MCSLOGDIR << "/trace/" << outName << '.' << sessionID;
#endif #endif
oFile.reset(new ofstream()); oFile.reset(new ofstream());
oFile->open(oss.str().c_str(), ios_base::out | ios_base::ate | ios_base::app); oFile->open(oss.str().c_str(), ios_base::out | ios_base::ate | ios_base::app);

View File

@ -45,6 +45,7 @@ typedef int pthread_t;
#include "errorcodes.h" #include "errorcodes.h"
#include "calpontsystemcatalog.h" #include "calpontsystemcatalog.h"
#include "blockcacheclient.h" #include "blockcacheclient.h"
#include "mcsconfig.h"
using namespace messageqcpp; using namespace messageqcpp;
using namespace std; using namespace std;
@ -256,7 +257,7 @@ int BPPSeeder::operator()()
#ifdef _MSC_VER #ifdef _MSC_VER
LogFileName << "C:/Calpont/log/trace/pt." << tid; LogFileName << "C:/Calpont/log/trace/pt." << tid;
#else #else
LogFileName << "/var/log/mariadb/columnstore/trace/pt." << tid; LogFileName << MCSLOGDIR << "/trace/pt." << tid;
#endif #endif
spof.reset(new PTLogs_t(gThdCnt, LogFileName.str().c_str())); spof.reset(new PTLogs_t(gThdCnt, LogFileName.str().c_str()));
gThdCnt++; gThdCnt++;

View File

@ -331,16 +331,12 @@ void* waitForSIGUSR1(void* p)
if (rec_sig == SIGUSR1) if (rec_sig == SIGUSR1)
{ {
#ifdef _MSC_VER ostringstream out;
ofstream out("C:/Calpont/log/trace/pplru.dat");
#else
ofstream out("/var/log/mariadb/columnstore/trace/pplru.dat");
#endif
for (int i = 0; i < cacheCount; i++) for (int i = 0; i < cacheCount; i++)
{ {
BRPp[i]->formatLRUList(out); BRPp[i]->formatLRUList(out);
out << "###" << endl; cout << out.str() << "###" << endl;
} }
} }
else if (rec_sig == SIGUSR2) else if (rec_sig == SIGUSR2)

View File

@ -23,12 +23,13 @@
#include <signal.h> #include <signal.h>
#include <sys/types.h> #include <sys/types.h>
#include <unistd.h> #include <unistd.h>
#include "mcsconfig.h"
void fatalHandler(int sig) void fatalHandler(int sig)
{ {
char filename[128]; char filename[128];
void* addrs[128]; void* addrs[128];
snprintf(filename, 128, "/var/log/mariadb/columnstore/trace/%s.%d.log", program_invocation_short_name, getpid()); snprintf(filename, 128, "%s/trace/%s.%d.log", MCSLOGDIR, program_invocation_short_name, getpid());
FILE* logfile = fopen(filename, "w"); FILE* logfile = fopen(filename, "w");
char s[30]; char s[30];
struct tm tim; struct tm tim;

View File

@ -97,7 +97,7 @@ public:
outName = name; outName = name;
ostringstream oss; ostringstream oss;
oss << "/var/log/mariadb/columnstore/trace/" << outName << '.' << sessionID; oss << MCSLOGDIR << "/trace/" << outName << '.' << sessionID;
oFile.reset(new ofstream()); oFile.reset(new ofstream());
oFile->open(oss.str().c_str(), ios_base::out | ios_base::ate | ios_base::app); oFile->open(oss.str().c_str(), ios_base::out | ios_base::ate | ios_base::app);
} }

View File

@ -33,6 +33,7 @@ using namespace logging;
#include <iomanip> #include <iomanip>
#include <sstream> #include <sstream>
#include "boost/date_time/posix_time/posix_time_types.hpp" #include "boost/date_time/posix_time/posix_time_types.hpp"
#include "mcsconfig.h"
namespace threadpool namespace threadpool
{ {
@ -523,7 +524,7 @@ void ThreadPool::dump()
void ThreadPoolMonitor::operator()() void ThreadPoolMonitor::operator()()
{ {
ostringstream filename; ostringstream filename;
filename << "/var/log/mariadb/columnstore/trace/ThreadPool_" << fPool->name() << ".log"; filename << MCSLOGDIR << "/trace/ThreadPool_" << fPool->name() << ".log";
fLog = new ofstream(filename.str().c_str()); fLog = new ofstream(filename.str().c_str());
for (;;) for (;;)

View File

@ -36,12 +36,14 @@
#include "tracer.h" #include "tracer.h"
#undef TRACER_DLLEXPORT #undef TRACER_DLLEXPORT
#include "mcsconfig.h"
using namespace std; using namespace std;
namespace BRM namespace BRM
{ {
#ifdef BRM_INFO #ifdef BRM_INFO
std::ofstream brmlog("/var/log/mariadb/columnstore/brm.log", std::ios::app); std::ofstream brmlog(string(MCSLOGDIR) + "/brm.log", std::ios::app);
Tracer::Tracer(const std::string& file, int line, const std::string& msg, bool debug, bool writeNow): fFileName(file), fLine(line), fMsg(msg), fDebug(debug), fpid(getpid()) Tracer::Tracer(const std::string& file, int line, const std::string& msg, bool debug, bool writeNow): fFileName(file), fLine(line), fMsg(msg), fDebug(debug), fpid(getpid())
{ {

View File

@ -52,6 +52,7 @@
#include "IDBPolicy.h" #include "IDBPolicy.h"
#include "MonitorProcMem.h" #include "MonitorProcMem.h"
#include "dataconvert.h" #include "dataconvert.h"
#include "mcsconfig.h"
using namespace std; using namespace std;
using namespace WriteEngine; using namespace WriteEngine;
@ -1149,7 +1150,8 @@ int main(int argc, char** argv)
int rc = NO_ERROR; int rc = NO_ERROR;
std::string exceptionMsg; std::string exceptionMsg;
TASK task; // track tasks being performed TASK task; // track tasks being performed
// set this upfront
curJob.setErrorDir(string(MCSLOGDIR) + "/cpimport/");
try try
{ {
//-------------------------------------------------------------------------- //--------------------------------------------------------------------------

View File

@ -53,6 +53,7 @@
#include "idbcompress.h" #include "idbcompress.h"
#include "calpontsystemcatalog.h" #include "calpontsystemcatalog.h"
#include "we_ddlcommandclient.h" #include "we_ddlcommandclient.h"
#include "mcsconfig.h"
using namespace std; using namespace std;
@ -76,7 +77,6 @@ namespace WriteEngine
/* static */ const std::string BulkLoad::DIR_BULK_JOB("job"); /* static */ const std::string BulkLoad::DIR_BULK_JOB("job");
/* static */ const std::string BulkLoad::DIR_BULK_TEMP_JOB("tmpjob"); /* static */ const std::string BulkLoad::DIR_BULK_TEMP_JOB("tmpjob");
/* static */ const std::string BulkLoad::DIR_BULK_IMPORT("/data/import/"); /* static */ const std::string BulkLoad::DIR_BULK_IMPORT("/data/import/");
/* static */ const std::string BulkLoad::DIR_BULK_LOG("/log/");
/* static */ bool BulkLoad::fNoConsoleOutput = false; /* static */ bool BulkLoad::fNoConsoleOutput = false;
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
@ -275,9 +275,9 @@ int BulkLoad::loadJobInfo(
const Job& curJob = fJobInfo.getJob(); const Job& curJob = fJobInfo.getJob();
string logFile, errlogFile; string logFile, errlogFile;
logFile = fRootDir + DIR_BULK_LOG + "Job_" + logFile = std::string(MCSLOGDIR) + "/cpimport/" + "Job_" +
Convertor::int2Str( curJob.id ) + LOG_SUFFIX; Convertor::int2Str( curJob.id ) + LOG_SUFFIX;
errlogFile = fRootDir + DIR_BULK_LOG + "Job_" + errlogFile = std::string(MCSLOGDIR) + "/cpimport/" + "Job_" +
Convertor::int2Str( curJob.id ) + ERR_LOG_SUFFIX; Convertor::int2Str( curJob.id ) + ERR_LOG_SUFFIX;
if (disableConsoleOutput()) if (disableConsoleOutput())
@ -1055,7 +1055,7 @@ int BulkLoad::processJob( )
(fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC)) (fBulkMode == BULK_MODE_REMOTE_MULTIPLE_SRC))
tableInfo->setBulkLoadMode( fBulkMode, fBRMRptFileName ); tableInfo->setBulkLoadMode( fBulkMode, fBRMRptFileName );
tableInfo->setErrorDir(getErrorDir()); tableInfo->setErrorDir(string(getErrorDir()));
tableInfo->setTruncationAsError(getTruncationAsError()); tableInfo->setTruncationAsError(getTruncationAsError());
rc = manageImportDataFileList( curJob, i, tableInfo ); rc = manageImportDataFileList( curJob, i, tableInfo );

View File

@ -1577,9 +1577,9 @@ void TableInfo::writeBadRows( const std::vector<std::string>* errorDatRows,
rejectFileNameToLog = fRejectDataFileName; rejectFileNameToLog = fRejectDataFileName;
} }
oss << "Number of rows with errors = " << fRejectDataCnt << oss << "Number of rows with bad data = " << fRejectDataCnt <<
". Exact error rows are listed in file " << ". Exact rows are listed in file located here: " <<
rejectFileNameToLog; fErrorDir;
fLog->logMsg(oss.str(), MSGLVL_INFO1); fLog->logMsg(oss.str(), MSGLVL_INFO1);
fRejectDataCnt = 0; fRejectDataCnt = 0;
@ -1691,9 +1691,9 @@ void TableInfo::writeErrReason( const std::vector< std::pair<RID,
errFileNameToLog = fRejectErrFileName; errFileNameToLog = fRejectErrFileName;
} }
oss << "Number of rows with errors = " << fRejectErrCnt << oss << "Number of rows with errors = " << fRejectDataCnt <<
". Row numbers with error reasons are listed in file " << ". Exact rows are listed in file located here: " <<
errFileNameToLog; fErrorDir;
fLog->logMsg(oss.str(), MSGLVL_INFO1); fLog->logMsg(oss.str(), MSGLVL_INFO1);
fRejectErrCnt = 0; fRejectErrCnt = 0;

View File

@ -77,7 +77,8 @@ WECmdArgs::WECmdArgs(int argc, char** argv) :
fbTruncationAsError(false), fbTruncationAsError(false),
fUUID(boost::uuids::nil_generator()()), fUUID(boost::uuids::nil_generator()()),
fConsoleOutput(true), fConsoleOutput(true),
fTimeZone("SYSTEM") fTimeZone("SYSTEM"),
fErrorDir(string(MCSLOGDIR)+"/cpimport/")
{ {
try try
{ {
@ -146,6 +147,9 @@ std::string WECmdArgs::getCpImportCmdLine()
aSS << " -f " << fPmFilePath; aSS << " -f " << fPmFilePath;
} }
if (fErrorDir.length() > 0)
aSS << " -L " << fErrorDir;
if (fUsername.length() > 0) if (fUsername.length() > 0)
aSS << " -U " << fUsername; aSS << " -U " << fUsername;
@ -567,7 +571,10 @@ void WECmdArgs::usage()
<< "\t-K\tS3 Authentication Secret (for S3 imports)\n" << "\t-K\tS3 Authentication Secret (for S3 imports)\n"
<< "\t-t\tS3 Bucket (for S3 imports)\n" << "\t-t\tS3 Bucket (for S3 imports)\n"
<< "\t-H\tS3 Hostname (for S3 imports, Amazon's S3 default)\n" << "\t-H\tS3 Hostname (for S3 imports, Amazon's S3 default)\n"
<< "\t-g\tS3 Region (for S3 imports)\n"; << "\t-g\tS3 Region (for S3 imports)\n"
<< "\t-L\tDirectory for the output .err and .bad files.\n"
<< "\t\tDefault is " << string(MCSLOGDIR);
cout << "\nExample1: Traditional usage\n" cout << "\nExample1: Traditional usage\n"
<< "\tcpimport -j 1234"; << "\tcpimport -j 1234";
@ -602,7 +609,7 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv)
fPrgmName = string(MCSBINDIR) + "/" + "cpimport.bin"; //argv[0] is splitter but we need cpimport fPrgmName = string(MCSBINDIR) + "/" + "cpimport.bin"; //argv[0] is splitter but we need cpimport
while ((aCh = getopt(argc, argv, while ((aCh = getopt(argc, argv,
"d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:Ny:K:t:H:g:U:")) "d:j:w:s:v:l:r:b:e:B:f:q:ihm:E:C:P:I:n:p:c:ST:Ny:K:t:H:g:U:L:"))
!= EOF) != EOF)
{ {
switch (aCh) switch (aCh)
@ -942,6 +949,12 @@ void WECmdArgs::parseCmdLineArgs(int argc, char** argv)
break; break;
} }
case 'L': // -L set the output location of .bad/.err files
{
fErrorDir = optarg;
break;
}
default: default:
{ {
std::string aErr = "Unknown command line option " + aCh; std::string aErr = "Unknown command line option " + aCh;

View File

@ -268,6 +268,14 @@ public:
{ {
return fS3Region; return fS3Region;
} }
std::string getErrorDir() const
{
return fErrorDir;
}
void setErrorDir(std::string fErrorDir)
{
this->fErrorDir = fErrorDir;
}
std::string& getUsername(); std::string& getUsername();
std::string PrepMode2ListOfFiles(std::string& FileName); // Bug 4342 std::string PrepMode2ListOfFiles(std::string& FileName); // Bug 4342
void getColumnList( std::set<std::string>& columnList ) const; void getColumnList( std::set<std::string>& columnList ) const;
@ -329,6 +337,7 @@ private: // variables for SplitterApp
bool fConsoleOutput; // If false, no output to console. bool fConsoleOutput; // If false, no output to console.
std::string fTimeZone; // Timezone to use for TIMESTAMP datatype std::string fTimeZone; // Timezone to use for TIMESTAMP datatype
std::string fUsername; // Username of the data files owner std::string fUsername; // Username of the data files owner
std::string fErrorDir;
}; };
//---------------------------------------------------------------------- //----------------------------------------------------------------------

View File

@ -474,20 +474,20 @@ void WESDHandler::setup()
std::string aErrLogName; std::string aErrLogName;
if (fRef.fCmdArgs.isJobLogOnly()) if (fRef.fCmdArgs.isJobLogOnly())
aLogName = fRef.fCmdArgs.getBulkRootDir() + "/log/" + aLogName = std::string(MCSLOGDIR) + "/cpimport/" +
"cpimport_Job_" + fRef.fCmdArgs.getJobId() + ".log"; "cpimport_Job_" + fRef.fCmdArgs.getJobId() + ".log";
else else
aLogName = fRef.fCmdArgs.getBulkRootDir() + "/log/" + aLogName = std::string(MCSLOGDIR) + "/cpimport/" +
"cpimport_" + aTimeStamp + "_" + aPid.str() + ".log"; "cpimport_" + aTimeStamp + "_" + aPid.str() + ".log";
if (getDebugLvl() > 1) cout << "LogName : " << aLogName << endl; if (getDebugLvl() > 1) cout << "LogName : " << aLogName << endl;
if (fRef.fCmdArgs.isJobLogOnly()) if (fRef.fCmdArgs.isJobLogOnly())
aErrLogName = fRef.fCmdArgs.getBulkRootDir() + "/log/" + aErrLogName = std::string(MCSLOGDIR) + "/cpimport/" +
"cpimport_Job_" + fRef.fCmdArgs.getJobId() + ".err"; "cpimport_Job_" + fRef.fCmdArgs.getJobId() + ".err";
else else
aErrLogName = fRef.fCmdArgs.getBulkRootDir() + "/log/" + aErrLogName = std::string(MCSLOGDIR) + "/cpimport/" +
"cpimport_" + aTimeStamp + "_" + aPid.str() + ".err"; "cpimport_" + aTimeStamp + "_" + aPid.str() + ".err";
@ -1529,7 +1529,7 @@ void WESDHandler::onErrorFile(int PmId, messageqcpp::SBS& Sbs)
{ {
std::string aFile = aTmpFileName.substr(aPos + 1); //+1 to pass '/' std::string aFile = aTmpFileName.substr(aPos + 1); //+1 to pass '/'
std::string aInFile = fRef.getLocFile(); // input file std::string aInFile = fRef.fCmdArgs.getErrorDir();
if (aInFile != "/dev/stdin") if (aInFile != "/dev/stdin")
{ {
@ -1627,7 +1627,7 @@ void WESDHandler::onBadFile(int PmId, messageqcpp::SBS& Sbs)
{ {
std::string aFile = aTmpFileName.substr(aPos + 1); //+1 to pass '/' std::string aFile = aTmpFileName.substr(aPos + 1); //+1 to pass '/'
std::string aInFile = fRef.getLocFile(); // input file std::string aInFile = fRef.fCmdArgs.getErrorDir();
if (aInFile != "/dev/stdin") if (aInFile != "/dev/stdin")
{ {

View File

@ -29,6 +29,7 @@
#include "we_config.h" #include "we_config.h"
#include "we_xmltag.h" #include "we_xmltag.h"
#include "we_xmlgendata.h" #include "we_xmlgendata.h"
#include "mcsconfig.h"
#include <boost/date_time/posix_time/posix_time.hpp> #include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/filesystem/path.hpp> #include <boost/filesystem/path.hpp>
@ -39,7 +40,6 @@ namespace
const char* DICT_TYPE("D"); const char* DICT_TYPE("D");
const char* ENCODING("UTF-8"); const char* ENCODING("UTF-8");
const char* LOGNAME("Jobxml_"); const char* LOGNAME("Jobxml_");
const std::string LOGDIR("/log/");
const char* JOBNAME("Job_"); const char* JOBNAME("Job_");
} }
@ -60,9 +60,9 @@ XMLGenProc::XMLGenProc(XMLGenData* mgr, bool bUseXmlLogFile, bool bSysCatRpt) :
fSysCatRpt(bSysCatRpt), fSysCatRpt(bSysCatRpt),
fUseXmlLogFile(bUseXmlLogFile) fUseXmlLogFile(bUseXmlLogFile)
{ {
std::string logFile(Config::getBulkRoot() + std::string(LOGDIR) + LOGNAME + std::string logFile(std::string(MCSLOGDIR) + "/cpimport/" + LOGNAME +
fInputMgr->getParm(XMLGenData::JOBID) + ".log" ); fInputMgr->getParm(XMLGenData::JOBID) + ".log" );
std::string errFile(Config::getBulkRoot() + std::string(LOGDIR) + LOGNAME + std::string errFile(std::string(MCSLOGDIR) + "/cpimport/" + LOGNAME +
fInputMgr->getParm(XMLGenData::JOBID) + ".err" ); fInputMgr->getParm(XMLGenData::JOBID) + ".err" );
fErrorString.append(errFile + "\n"); fErrorString.append(errFile + "\n");