From ce10b08e8eed1e87a8e17a238682a58038f5447d Mon Sep 17 00:00:00 2001
From: david hill <david.hill@mariadb.com>
Date: Mon, 7 Nov 2016 16:53:09 -0600
Subject: [PATCH 01/54] change amazon scripts for centos 7 and mcs

---
 oam/cloud/MCSInstanceCmds.sh | 574 +++++++++++++++++++++++++++++++++++
 oam/cloud/MCSVolumeCmds.sh   | 337 ++++++++++++++++++++
 2 files changed, 911 insertions(+)
 create mode 100644 oam/cloud/MCSInstanceCmds.sh
 create mode 100644 oam/cloud/MCSVolumeCmds.sh

diff --git a/oam/cloud/MCSInstanceCmds.sh b/oam/cloud/MCSInstanceCmds.sh
new file mode 100644
index 000000000..40d58f355
--- /dev/null
+++ b/oam/cloud/MCSInstanceCmds.sh
@@ -0,0 +1,574 @@
+#! /bin/sh
+# IDBInstanceCmds.sh
+# get-local-instance-ID, get-zone, getPrivateIP from a Cloud environment
+#
+# 1. Amazon EC2
+
+prefix=/usr/local
+
+#check command
+if [ "$1" = "" ]; then
+	echo "Enter Command Name: {launchInstance|getInstance|getZone|getPrivateIP|getKey|getAMI|getType|terminateInstance|startInstance|assignElasticIP|deassignElasticIP|getProfile|stopInstance|getGroup|getSubnet}
+}"
+	exit 1
+fi
+
+if [ "$1" = "getPrivateIP" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Instance Name"
+		exit 1
+	fi
+	instanceName="$2"
+fi
+
+if [ "$1" = "launchInstance" ]; then
+	if [ "$2" = "" ]; then
+		IPaddress="unassigned"
+	else
+		IPaddress="$2"
+	fi
+	if [ "$3" = "" ]; then
+		instanceType="unassigned"
+	else
+		instanceType="$3"
+	fi
+	if [ "$4" = "" ]; then
+		group="unassigned"
+	else
+		group="$4"
+	fi
+fi
+
+if [ "$1" = "terminateInstance" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Instance Name"
+		exit 1
+	fi
+	instanceName="$2"
+fi
+
+if [ "$1" = "stopInstance" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Instance Name"
+		exit 1
+	fi
+	instanceName="$2"
+fi
+
+if [ "$1" = "startInstance" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Instance Name"
+		exit 1
+	fi
+	instanceName="$2"
+fi
+
+if [ "$1" = "assignElasticIP" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Instance Name"
+		exit 1
+	else
+		instanceName="$2"
+	fi
+	if [ "$3" = "" ]; then
+		echo "Enter Elastic IP Address"
+		exit 1
+	else
+		IPAddress="$3"
+	fi
+fi
+
+if [ "$1" = "deassignElasticIP" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Elastic IP Address"
+		exit 1
+	else
+		IPAddress="$2"
+	fi
+fi
+
+
+test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions
+
+ec2=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_HOME`
+
+if [ $ec2 == "unassigned" ]; then
+	if [ "$1" = "getPrivateIP" ]; then
+		echo "stopped"
+		exit 1
+	else
+       echo "unknown"
+        exit 1
+	fi
+fi
+
+java=`$prefix/mariadb/columnstore/bin/getConfig Installation JAVA_HOME`
+path=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_PATH`
+
+export PATH=$path
+export EC2_HOME=$ec2
+export JAVA_HOME=$java
+
+# get Keys and region
+AmazonAccessKeyFile=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonAccessKey`
+if [ $AmazonAccessKeyFile == "unassigned" ]; then
+	echo "FAILED: missing Config Setting AmazonAccessKey : $AmazonAccessKeyfile"
+	exit 1
+fi
+
+AmazonSecretKeyFile=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonSecretKey`
+if [ $AmazonSecretKeyFile == "unassigned" ]; then
+	echo "FAILED: missing Config Setting AmazonSecretKeyFile : $AmazonSecretKeyFile"
+	exit 1
+fi
+
+AmazonAccessKey=`cat $AmazonAccessKeyFile`
+AmazonSecretKey=`cat $AmazonSecretKeyFile`
+
+Region=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonRegion`
+subnet=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonSubNetID`
+
+if test ! -f $AmazonAccessKeyfile ; then
+	echo "FAILED: missing AmazonAccessKeyfile : $AmazonAccessKeyfile"
+	exit 1
+fi
+
+if test ! -f $AmazonSecretKeyfile ; then
+	echo "FAILED: missing AmazonSecretKeyfile : $AmazonSecretKeyfile"
+	exit 1
+fi
+
+#default instance to null
+instance=""
+
+describeInstanceFile="/tmp/describeInstance.txt"
+touch $describeInstanceFile
+
+describeInstance() {
+	ec2-describe-instances -O $AmazonAccessKey -W $AmazonSecretKey --region $Region > $describeInstanceFile 2>&1
+}
+
+#call at start up
+describeInstance
+
+getInstance() {
+	if [ "$instance" != "" ]; then
+		echo $instance
+		return
+	fi
+	
+	# first get local IP Address
+	localIP=`ifconfig eth0 | grep -m 1 "inet" | awk '{print substr($2,0,20)}'`
+
+	#get local Instance ID
+	instance=`cat $describeInstanceFile | grep -m 1 -w $localIP |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+	if [ "$instance" == "" ]; then
+		describeInstance
+	fi
+	instance=`cat $describeInstanceFile | grep -m 1 -w $localIP |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+
+	echo $instance
+	return
+}
+
+getInstancePrivate() {
+	if [ "$instance" != "" ]; then
+		echo $instance
+		return
+	fi
+	
+	# first get local IP Address
+        localIP=`ifconfig eth0 | grep -m 1 "inet" | awk '{print substr($2,0,20)}'`
+
+	#get local Instance ID
+	instance=`cat $describeInstanceFile | grep -m 1 -w $localIP |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+	if [ "$instance" == "" ]; then
+		describeInstance
+	fi
+	instance=`cat $describeInstanceFile | grep -m 1 -w $localIP |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+
+	return
+}
+
+
+getZone() {
+	#get from Columnstore.xml if it's there, if not, get from instance then store
+	zone=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonZone`
+
+	if [ "$zone" = "unassigned" ] || [ "$zone" = "" ]; then
+		#get local Instance ID
+		getInstancePrivate >/dev/null 2>&1
+		#get zone
+		if [ "$subnet" == "unassigned" ]; then
+			zone=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $11}'`
+			if [ "$zone" == "" ]; then
+				describeInstance
+			fi
+			zone=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $11}'`
+
+		else
+			zone=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $11}'`
+			if [ "$zone" == "" ]; then
+				describeInstance
+			fi
+			zone=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $11}'`
+		fi
+		$prefix/mariadb/columnstore/bin/setConfig Installation AmazonZone $zone
+	fi
+
+	echo $zone
+	return
+}
+
+getPrivateIP() {
+	#get instance info
+	grep -B1 -A4 -m 1 $instanceName $describeInstanceFile > /tmp/instanceInfo_$instanceName 2>&1
+	if [ `cat /tmp/instanceInfo_$instanceName | wc -c` -eq 0 ]; then
+		describeInstance
+	fi
+	grep -B1 -A4 -m 1 $instanceName $describeInstanceFile > /tmp/instanceInfo_$instanceName 2>&1
+
+	#check if running or terminated
+	cat /tmp/instanceInfo_$instanceName | grep running > /tmp/instanceStatus_$instanceName
+	if [ `cat /tmp/instanceStatus_$instanceName | wc -c` -eq 0 ]; then
+		# not running
+		cat /tmp/instanceInfo_$instanceName | grep pending > /tmp/instanceStatus_$instanceName
+		if [ `cat /tmp/instanceStatus_$instanceName | wc -c` -ne 0 ]; then
+			describeInstance
+			echo "stopped"
+			exit 1
+		else
+			cat /tmp/instanceInfo_$instanceName | grep terminated > /tmp/instanceStatus_$instanceName
+			if [ `cat /tmp/instanceStatus_$instanceName | wc -c` -ne 0 ]; then
+				echo "terminated"
+				exit 1
+			else
+				cat /tmp/instanceInfo_$instanceName | grep shutting-down > /tmp/instanceStatus_$instanceName
+				if [ `cat /tmp/instanceStatus_$instanceName | wc -c` -ne 0 ]; then
+					echo "terminated"
+					exit 1
+				else
+					echo "stopped"
+					exit 1
+				fi
+			fi
+		fi
+	fi
+	
+	#running, get priviate IP Address
+	if [ "$subnet" == "unassigned" ]; then
+		IpAddr=`head -n 2 /tmp/instanceInfo_$instanceName | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $14}'`
+	else
+		IpAddr=`head -n 2 /tmp/instanceInfo_$instanceName | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $13}'`
+	fi
+
+	echo $IpAddr
+	exit 0
+}
+
+getType() {
+	#get local Instance ID
+	getInstancePrivate >/dev/null 2>&1
+	#get Type
+	if [ "$subnet" == "unassigned" ]; then
+		instanceType=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $9}'`
+		if [ "$instanceType" == "" ]; then
+			describeInstance
+		fi
+		instanceType=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $9}'`
+
+	else
+		instanceType=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $9}'`
+		if [ "$instanceType" == "" ]; then
+			describeInstance
+		fi
+		instanceType=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $9}'`
+
+	fi
+
+	echo $instanceType
+	return
+}
+
+getKey() {
+	#get local Instance ID
+	getInstancePrivate >/dev/null 2>&1
+	#get Key
+	if [ "$subnet" == "unassigned" ]; then
+		key=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $7}'`
+		if [ "$key" == "" ]; then
+			describeInstance
+		fi
+		key=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $7}'`
+
+	else
+		key=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $7}'`
+		if [ "$key" == "" ]; then
+			describeInstance
+		fi
+		key=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $7}'`
+
+	fi
+
+	echo $key
+	return
+}
+
+getAMI() {
+	#get local Instance ID
+	getInstancePrivate >/dev/null 2>&1
+	#get AMI
+	ami=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $3}'`
+	if [ "$ami" == "" ]; then
+		describeInstance
+	fi
+	ami=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $3}'`
+
+	echo $ami
+	return
+}
+
+getGroup() {
+	#get local Instance ID
+	getInstancePrivate >/dev/null 2>&1
+	#get group 
+	if [ "$subnet" == "unassigned" ]; then
+		group=`grep -B1 -A4 -m 1 $instance $describeInstanceFile |  grep -m 1 RESERVATION | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $4}'`
+		if [ "$group" == "" ]; then
+			describeInstance
+		fi
+		group=`grep -B1 -A4 -m 1 $instance $describeInstanceFile |  grep -m 1 RESERVATION | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $4}'`
+		if [ "$group" == "" ]; then
+			group=`grep -B1 -A4 -m 1 $instance $describeInstanceFile |  grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $21}'`
+		fi
+	else
+		group=`grep -B1 -A6 -m 1 $instance $describeInstanceFile |  grep -m 1 GROUP | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+		if [ "$group" == "" ]; then
+			describeInstance
+		fi
+		group=`grep -B1 -A6 -m 1 $instance $describeInstanceFile |  grep -m 1 GROUP | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+		if [ "$group" == "" ]; then
+			group=`grep -B1 -A4 -m 1 $instance $describeInstanceFile |  grep -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $21}'`
+		fi
+	fi
+
+	echo $group
+	return
+}
+
+getProfile() {
+	#get local Instance ID
+	getInstancePrivate >/dev/null 2>&1
+	#get Type
+	if [ "$subnet" == "unassigned" ]; then
+		instanceProfile=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $23}'`
+		if [ "$instanceProfile" == "" ]; then
+			describeInstance
+		fi
+		instanceProfile=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $23}'`
+
+	else
+		instanceProfile=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $22}'`
+		if [ "$instanceProfile" == "" ]; then
+			describeInstance
+		fi
+		instanceProfile=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $22}'`
+
+	fi
+
+	echo $instanceProfile
+	return
+}
+
+launchInstance() {
+	#get publickey
+	getKey >/dev/null 2>&1
+	if [ "$group" = "unassigned" ]; then
+		#get group
+		getGroup >/dev/null 2>&1
+	fi
+	#get AMI
+	getAMI >/dev/null 2>&1
+	#get Zone
+	getZone >/dev/null 2>&1
+	if [ "$instanceType" = "unassigned" ]; then
+		#get type
+		getType >/dev/null 2>&1
+	fi
+	#get AMI Profile
+	getProfile >/dev/null 2>&1
+
+	if [ "$subnet" == "unassigned" ]; then
+		#NOT VPC
+		if [ "$instanceProfile" = "" ] || [ "$instanceProfile" = "default" ]; then
+			newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone --region $Region $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+		else
+			newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone -p $instanceProfile --region $Region $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+		fi
+	else	# VPC
+		if [ "$instanceProfile" = "" ] || [ "$instanceProfile" = "default" ]; then
+			if [ "$group" != "default" ]; then
+				if [ "$IPaddress" = "autoassign" ]; then
+					newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone  --region $Region -s $subnet $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+				else
+					newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone  --region $Region -s $subnet --private-ip-address $IPaddress $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+				fi
+			else
+				if [ "$IPaddress" = "autoassign" ]; then
+					newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -t $instanceType -z $zone --region $Region -s $subnet $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+				else
+					newInstance=`ec2-run-instances --O $AmazonAccessKey -W $AmazonSecretKey -k $key -t $instanceType -z $zone --region $Region -s $subnet --private-ip-address $IPaddress $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+				fi
+			fi
+		else
+			if [ "$group" != "default" ]; then
+				if [ "$IPaddress" = "autoassign" ]; then
+					newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone -p $instanceProfile --region $Region -s $subnet $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+				else
+					newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -g $group -t $instanceType -z $zone -p $instanceProfile --region $Region -s $subnet --private-ip-address $IPaddress $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+				fi
+			else
+				if [ "$IPaddress" = "autoassign" ]; then
+					newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -t $instanceType -z $zone -p $instanceProfile --region $Region -s $subnet $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+				else
+					newInstance=`ec2-run-instances -O $AmazonAccessKey -W $AmazonSecretKey -k $key -t $instanceType -z $zone -p $instanceProfile --region $Region -s $subnet --private-ip-address $IPaddress $ami | grep  -m 1 INSTANCE | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+				fi
+			fi
+		fi
+	fi
+	echo $newInstance
+	return
+}
+
+terminateInstance() {
+	#terminate Instance
+	ec2-terminate-instances -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $instanceName > /tmp/termInstanceInfo_$instanceName 2>&1
+	return
+}
+
+stopInstance() {
+	#terminate Instance
+	ec2-stop-instances -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $instanceName > /tmp/stopInstanceInfo_$instanceName 2>&1
+	return
+}
+
+startInstance() {
+	#terminate Instance
+	ec2-start-instances -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $instanceName > /tmp/startInstanceInfo_$instanceName 2>&1
+
+	cat /tmp/startInstanceInfo_$instanceName | grep INSTANCE > /tmp/startInstanceStatus_$instanceName
+	if [ `cat /tmp/startInstanceStatus_$instanceName | wc -c` -eq 0 ]; then
+		echo "Failed, check /tmp/startInstanceInfo_$instanceName"
+		exit 1
+	fi
+	echo "Success"
+	exit 0
+}
+
+assignElasticIP() {
+	#terminate Instance
+
+        if [ "$subnet" == "unassigned" ]; then
+		ec2-associate-address -O $AmazonAccessKey -W $AmazonSecretKey -i $instanceName $IPAddress > /tmp/assignElasticIPInfo_$IPAddress 2>&1
+	else
+		EIP=`ec2-describe-addresses -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $IPAddress | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $4}'`
+                ec2-associate-address -O $AmazonAccessKey -W $AmazonSecretKey --region $Region -i $instanceName -a $EIP > /tmp/assignElasticIPInfo_$IPAddress 2>&1
+	fi
+
+	cat /tmp/assignElasticIPInfo_$IPAddress | grep ADDRESS > /tmp/assignElasticIPStatus_$IPAddress
+	if [ `cat /tmp/assignElasticIPStatus_$IPAddress | wc -c` -eq 0 ]; then
+		echo "Failed, check /tmp/assignElasticIPInfo_$IPAddress"
+		exit 1
+	fi
+
+	echo "Success"
+	exit 0
+}
+
+deassignElasticIP() {
+	#terminate Instance
+	ec2-disassociate-address -O $AmazonAccessKey -W $AmazonSecretKey $IPAddress > /tmp/deassignElasticIPInfo_$IPAddress 2>&1
+
+	cat /tmp/deassignElasticIPInfo_$IPAddress | grep ADDRESS > /tmp/deassignElasticIPStatus_$IPAddress
+	if [ `cat /tmp/deassignElasticIPStatus_$IPAddress | wc -c` -eq 0 ]; then
+		echo "Failed, check /tmp/deassignElasticIPStatus_$IPAddress"
+		exit 1
+	fi
+
+	echo "Success"
+	exit 0
+}
+
+getSubnet() {
+        #get local Instance ID
+        getInstancePrivate >/dev/null 2>&1
+        #get Subnet
+        subnet=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $16}'`
+        if [ "$subnet" == "" ]; then
+                describeInstance
+        fi
+        subnet=`cat $describeInstanceFile | grep -m 1 $instance |  awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $16}'`
+
+	if [[ $subnet == *"subnet"* ]]
+	then
+        	echo $subnet
+	else
+		echo "failed"
+	fi
+
+        return
+}
+
+
+case "$1" in
+  getInstance)
+  	getInstance
+	;;
+  getZone)
+  	getZone
+	;;
+  getPrivateIP)
+  	getPrivateIP
+	;;
+  getKey)
+  	getKey
+	;;
+  getAMI)
+  	getAMI
+	;;
+  getType)
+  	getType
+	;;
+  launchInstance)
+  	launchInstance
+	;;
+  terminateInstance)
+  	terminateInstance
+	;;
+  stopInstance)
+  	stopInstance
+	;;
+  startInstance)
+  	startInstance
+	;;
+  assignElasticIP)
+  	assignElasticIP
+	;;
+  deassignElasticIP)
+  	deassignElasticIP
+	;;
+  getProfile)
+  	getProfile
+	;;
+  getGroup)
+  	getGroup
+	;;
+  getSubnet)
+  	getSubnet
+	;;
+  *)
+	echo $"Usage: $0 {launchInstance|getInstance|getZone|getPrivateIP|getType|getKey|getAMI|terminateInstance|startInstance|assignElasticIP|deassignElasticIP|getProfile|stopInstance|getGroup|getSubnet}"
+	exit 1
+esac
+
+exit $?
diff --git a/oam/cloud/MCSVolumeCmds.sh b/oam/cloud/MCSVolumeCmds.sh
new file mode 100644
index 000000000..3d64007b1
--- /dev/null
+++ b/oam/cloud/MCSVolumeCmds.sh
@@ -0,0 +1,337 @@
+#! /bin/sh
+# IDBVolumeCmds.sh
+# describe, detach and attach Volume Storage from a Cloud environment
+#
+# 1. Amazon EC2
+
+prefix=/usr/local
+
+#check command
+if [ "$1" = "" ]; then
+	echo "Enter Command Name: {create|describe|detach|attach|delete|createTag}"
+	exit 1
+fi
+
+if [ "$1" = "create" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter size of the volume, in GiB (1-1024)"
+		exit 1
+	fi
+	volumeSize="$2"
+
+	#get module-type
+	if [ "$3" = "" ]; then
+		echo "Enter Module Type"
+		exit 1
+	fi
+	moduleType="$3"
+fi
+
+if [ "$1" = "describe" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Volume Name"
+		exit 1
+	fi
+	volumeName="$2"
+fi
+
+if [ "$1" = "detach" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Volume Name"
+		exit 1
+	fi
+	volumeName="$2"
+fi
+
+if [ "$1" = "attach" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Volume Name"
+		exit 1
+	fi
+	volumeName="$2"
+
+	#get instance-name and device-name
+	if [ "$3" = "" ]; then
+		echo "Enter Instance Name"
+		exit 1
+	fi
+	instanceName="$3"
+
+	if [ "$4" = "" ]; then
+		echo "Enter Device Name"
+		exit 1
+	fi
+	deviceName="$4"
+fi
+
+if [ "$1" = "delete" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Volume Name"
+		exit 1
+	fi
+	volumeName="$2"
+fi
+
+if [ "$1" = "createTag" ]; then
+	if [ "$2" = "" ]; then
+		echo "Enter Resource Name"
+		exit 1
+	fi
+	resourceName="$2"
+
+	if [ "$3" = "" ]; then
+		echo "Enter Tag Name"
+		exit 1
+	fi
+	tagName="$3"
+
+	if [ "$4" = "" ]; then
+		echo "Enter Tag Value"
+		exit 1
+	fi
+	tagValue="$4"
+fi
+
+
+test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions
+
+ec2=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_HOME`
+
+if [ $ec2 == "unassigned" ]; then
+       STATUS="unknown"
+        RETVAL=1
+fi
+
+java=`$prefix/mariadb/columnstore/bin/getConfig Installation JAVA_HOME`
+path=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_PATH`
+
+export PATH=$path
+export EC2_HOME=$ec2
+export JAVA_HOME=$java
+
+# get Keys and region
+AmazonAccessKeyFile=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonAccessKey`
+if [ $AmazonAccessKeyFile == "unassigned" ]; then
+	echo "FAILED: missing Config Setting AmazonAccessKey : $AmazonAccessKeyfile"
+	exit 1
+fi
+
+AmazonSecretKeyFile=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonSecretKey`
+if [ $AmazonSecretKeyFile == "unassigned" ]; then
+	echo "FAILED: missing Config Setting AmazonSecretKeyFile : $AmazonSecretKeyFile"
+	exit 1
+fi
+
+AmazonAccessKey=`cat $AmazonAccessKeyFile`
+AmazonSecretKey=`cat $AmazonSecretKeyFile`
+
+if test ! -f $AmazonAccessKeyfile ; then
+	echo "FAILED: missing AmazonAccessKeyfile : $AmazonAccessKeyfile"
+	exit 1
+fi
+
+if test ! -f $AmazonSecretKeyfile ; then
+	echo "FAILED: missing AmazonSecretKeyfile : $AmazonSecretKeyfile"
+	exit 1
+fi
+
+Region=`$prefix/mariadb/columnstore/bin/getConfig Installation AmazonRegion`
+
+
+checkInfostatus() {
+	#check if attached
+	cat /tmp/volumeInfo_$volumeName | grep attached > /tmp/volumeStatus_$volumeName
+	if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then
+		STATUS="attached"
+		RETVAL=0
+		return
+	fi
+	#check if available
+	cat /tmp/volumeInfo_$volumeName | grep available > /tmp/volumeStatus_$volumeName
+	if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then
+		STATUS="available"
+		RETVAL=0
+		return
+	fi
+	#check if detaching
+	cat /tmp/volumeInfo_$volumeName | grep detaching > /tmp/volumeStatus_$volumeName
+	if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then
+		STATUS="detaching"
+		RETVAL=0
+		return
+	fi
+	#check if attaching
+	cat /tmp/volumeInfo_$volumeName | grep attaching > /tmp/volumeStatus_$volumeName
+	if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then
+		STATUS="attaching"
+		RETVAL=0
+		return
+	fi
+	#check if doesn't exist
+	cat /tmp/volumeInfo_$volumeName | grep "does not exist" > /tmp/volumeStatus_$volumeName
+	if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then
+		STATUS="does-not-exist"
+		RETVAL=1
+		return
+	fi
+	#check if reports already attached from attach command
+	cat /tmp/volumeInfo_$volumeName | grep "already attached" > /tmp/volumeStatus_$volumeName
+	if [ `cat /tmp/volumeStatus_$volumeName | wc -c` -ne 0 ]; then
+		STATUS="already-attached"
+		RETVAL=1
+		return
+	fi
+	#any other, unknown error
+	STATUS="unknown"
+	RETVAL=1
+	return
+}
+
+createvolume() {
+	# get zone
+	zone=`$prefix/mariadb/columnstore/bin/IDBInstanceCmds.sh getZone`
+
+	if [ $moduleType == "um" ]; then
+		# get type
+		volumeType=`$prefix/mariadb/columnstore/bin/getConfig Installation UMVolumeType`
+		if [ $volumeType == "io1" ]; then
+			# get IOPS
+			volumeIOPS=`$prefix/mariadb/columnstore/bin/getConfig Installation UMVolumeIOPS`
+		fi
+	else	# pm
+		# get type
+		volumeType=`$prefix/mariadb/columnstore/bin/getConfig Installation PMVolumeType`
+		if [ $volumeType == "io1" ]; then
+			# get IOPS
+			volumeIOPS=`$prefix/mariadb/columnstore/bin/getConfig Installation PMVolumeIOPS`
+		fi
+	fi
+
+	#create volume
+	if [ $volumeType == "io1" ]; then
+		volume=`ec2-create-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region -z $zone -s $volumeSize -t $volumeType -iops $volumeIOPS  | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+	else
+		volume=`ec2-create-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region -z $zone -s $volumeSize -t $volumeType | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $2}'`
+	fi
+
+	echo $volume
+	return
+}
+
+describevolume() {
+	#describe volume
+	ec2-describe-volumes -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName > /tmp/volumeInfo_$volumeName 2>&1
+
+	checkInfostatus
+	echo $STATUS
+	return
+}
+
+detachvolume() {
+	#detach volume
+	ec2-detach-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName > /tmp/volumeInfo_$volumeName 2>&1
+
+	checkInfostatus
+	if [ $STATUS == "detaching" ]; then
+		retries=1
+		while [ $retries -ne 60 ]; do
+			#retry until it's attached
+			ec2-detach-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName > /tmp/volumeInfo_$volumeName 2>&1
+		
+			checkInfostatus
+			if [ $STATUS == "available" ]; then
+				echo "available"
+				exit 0
+			fi
+			((retries++))
+			sleep 1
+		done
+		test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions
+		cplogger -w 100 "detachvolume failed: $STATUS"
+		echo "failed"
+		exit 1
+	fi
+
+	if [ $STATUS == "available" ]; then
+		echo "available"
+		exit 0
+	fi
+
+	test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions
+	cplogger -w 100 "detachvolume failed status: $STATUS"
+	echo $STATUS
+	exit 1
+}
+
+attachvolume() {
+
+	#detach volume
+	ec2-attach-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName -i $instanceName -d $deviceName > /tmp/volumeInfo_$volumeName 2>&1
+
+	checkInfostatus
+	if [ $STATUS == "attaching" -o $STATUS == "already-attached" ]; then
+		retries=1
+		while [ $retries -ne 60 ]; do
+			#check status until it's attached
+			describevolume
+			if [ $STATUS == "attached" ]; then
+				echo "attached"
+				exit 0
+			fi
+			((retries++))
+			sleep 1
+		done
+		test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions
+		cplogger -w 100 "attachvolume failed: $STATUS"
+		echo "failed"
+		exit 1
+	fi
+
+	if [ $STATUS == "attached" ]; then
+		echo "attached"
+		exit 0
+	fi
+
+	test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions
+	cplogger -w 100 "attachvolume failed: $STATUS"
+	echo $STATUS
+	exit 1
+}
+
+deletevolume() {
+	#delete volume
+	ec2-delete-volume -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $volumeName > /tmp/deletevolume_$volumeName 2>&1
+	return
+}
+
+createTag() {
+	#create tag
+	ec2-create-tags -O $AmazonAccessKey -W $AmazonSecretKey --region $Region $resourceName --tag $tagName=$tagValue > /tmp/createTag_$volumeName 2>&1
+	return
+}
+
+case "$1" in
+  create)
+  	createvolume
+	;;
+  describe)
+  	describevolume
+	;;
+  detach)
+  	detachvolume
+	;;
+  attach)
+  	attachvolume
+	;;
+  delete)
+  	deletevolume
+	;;
+  createTag)
+  	createTag
+	;;
+  *)
+	echo $"Usage: $0 {create|describe|detach|attach|delete|}"
+	exit 1
+esac
+
+exit $?

From 40c3d05d3a41a8d40363bd9f280cc2a7dd0eb080 Mon Sep 17 00:00:00 2001
From: Andrew Hutchings <andrew@linuxjedi.co.uk>
Date: Wed, 9 Nov 2016 13:52:24 +0000
Subject: [PATCH 02/54] Fix rpath settings

I accidentally used CMAKE_INSTALL_PREFIX which isn't always set for the
engine build. This patch uses INSTALL_ENGINE instead
---
 CMakeLists.txt | 41 ++++++++++++++++++++---------------------
 1 file changed, 20 insertions(+), 21 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2df76b54b..82a372f7b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -36,26 +36,6 @@ ENDIF(NOT CMAKE_BUILD_TYPE)
 
 #set( CMAKE_VERBOSE_MAKEFILE on )
 
-# use, i.e. don't skip the full RPATH for the build tree
-SET(CMAKE_SKIP_BUILD_RPATH  FALSE)
-
-# when building, don't use the install RPATH already
-# (but later on when installing)
-SET(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
-
-SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
-
-# add the automatically determined parts of the RPATH
-# which point to directories outside the build tree to the install RPATH
-SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
-
-
-# the RPATH to be used when installing, but only if it's not a system directory
-LIST(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/lib" isSystemDir)
-IF("${isSystemDir}" STREQUAL "-1")
-   SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
-ENDIF("${isSystemDir}" STREQUAL "-1")
-
 SET (PACKAGE columnstore)
 SET (VERSION 1.0.2)
 SET (PACKAGE_NAME columnstore)
@@ -68,6 +48,25 @@ SET (PACKAGE_URL "")
 SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 SET (INSTALL_ENGINE "/usr/local/mariadb/columnstore")
 
+# use, i.e. don't skip the full RPATH for the build tree
+SET(CMAKE_SKIP_BUILD_RPATH  FALSE)
+
+# when building, don't use the install RPATH already
+# (but later on when installing)
+SET(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
+
+SET(CMAKE_INSTALL_RPATH "${INSTALL_ENGINE}/lib")
+
+# add the automatically determined parts of the RPATH
+# which point to directories outside the build tree to the install RPATH
+SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+
+# the RPATH to be used when installing, but only if it's not a system directory
+LIST(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${INSTALL_ENGINE}/lib" isSystemDir)
+IF("${isSystemDir}" STREQUAL "-1")
+    SET(CMAKE_INSTALL_RPATH "${INSTALL_ENGINE}/lib")
+ENDIF("${isSystemDir}" STREQUAL "-1")
+
 INCLUDE (configureEngine.cmake)
 
 CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/build/releasenum.in ${CMAKE_CURRENT_BINARY_DIR}/build/releasenum @ONLY IMMEDIATE)
@@ -110,7 +109,7 @@ ENDFOREACH()
 SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -ggdb3 -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D_DEBUG -DHAVE_CONFIG_H")
 SET(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -ggdb3 -fno-tree-vectorize -DSAFE_MUTEX -DSAFEMALLOC -DENABLED_DEBUG_SYNC -O0 -Wall -D _DEBUG -DHAVE_CONFIG_H")
 
-SET (ENGINE_LDFLAGS    "-Wl,--rpath -Wl,${INSTALL_ENGINE}/lib -Wl,--no-as-needed -Wl,--add-needed")  
+SET (ENGINE_LDFLAGS    "-Wl,--no-as-needed -Wl,--add-needed")  
 
 FIND_PACKAGE(Boost 1.53.0 REQUIRED COMPONENTS system filesystem thread regex date_time) 
 

From 461052d206a3df88fece4db441b65937a3cb116d Mon Sep 17 00:00:00 2001
From: Andrew Hutchings <andrew@linuxjedi.co.uk>
Date: Wed, 9 Nov 2016 14:02:01 +0000
Subject: [PATCH 03/54] Update MariaDB version number

---
 README    | 2 +-
 README.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README b/README
index b56720e1e..85722e3c3 100644
--- a/README
+++ b/README
@@ -1,6 +1,6 @@
 This is MariaDB ColumnStore 1.0.4
 MariaDB ColumnStore 1.0.4 is the development version of MariaDB ColumnStore. 
-It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.18 and adding entirely 
+It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely 
 new features not found anywhere else.
 
 MariaDB ColumnStore 1.0.4 is an Beta release. This is the first MariaDB 
diff --git a/README.md b/README.md
index 3754dfc2d..da3148b24 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 #MariaDB ColumnStore Storage/Execution engine 1.0.4
 MariaDB ColumnStore 1.0.4 is the development version of MariaDB ColumnStore. 
-It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.18 and adding entirely 
+It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely 
 new features not found anywhere else.
 
 #MariaDB ColumnStore 1.0.4 is an Beta release. 

From 42cd2032657383c4ca6ec82878a6b21602377cb0 Mon Sep 17 00:00:00 2001
From: Ben Thompson <dl3hp33ui@gmail.com>
Date: Wed, 9 Nov 2016 13:13:42 -0600
Subject: [PATCH 04/54] MCOL-372: Modify condition which message is logged.
 Before was logging once a second on every timeout.

---
 utils/messageqcpp/inetstreamsocket.cpp | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/utils/messageqcpp/inetstreamsocket.cpp b/utils/messageqcpp/inetstreamsocket.cpp
index b298843a2..57c863aea 100644
--- a/utils/messageqcpp/inetstreamsocket.cpp
+++ b/utils/messageqcpp/inetstreamsocket.cpp
@@ -436,10 +436,16 @@ const SBS InetStreamSocket::read(const struct ::timespec* timeout, bool* isTimeO
 	uint8_t* msglenp = reinterpret_cast<uint8_t*>(&msglen);
 	size_t mlread = 0;
 	
-	if (readToMagic(msecs, isTimeOut, stats) == false)	//indicates a timeout or EOF
+	bool myIsTimeOut = false;
+	if (readToMagic(msecs, &myIsTimeOut, stats) == false)	//indicates a timeout or EOF
 	{
-		logIoError("InetStreamSocket::read: timeout during readToMagic", 0);
-		return SBS(new ByteStream(0));
+	    if (!myIsTimeOut)
+	        logIoError("InetStreamSocket::read: EOF during readToMagic", 0);
+	    if (isTimeOut)
+	    {
+	        *isTimeOut = myIsTimeOut;
+	    }
+	    return SBS(new ByteStream(0));
 	}
 
 	//FIXME: This seems like a lot of work to read 4 bytes...

From cdc17de55127eff7dad13c56d898419f5825d8d8 Mon Sep 17 00:00:00 2001
From: david hill <david.hill@mariadb.com>
Date: Wed, 9 Nov 2016 13:43:15 -0600
Subject: [PATCH 05/54] update CMakeLists.txt

---
 oam/cloud/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oam/cloud/CMakeLists.txt b/oam/cloud/CMakeLists.txt
index 6e6469b41..8c18ca72c 100644
--- a/oam/cloud/CMakeLists.txt
+++ b/oam/cloud/CMakeLists.txt
@@ -1,2 +1,2 @@
 
-install(PROGRAMS  IDBInstanceCmds.sh IDBVolumeCmds.sh DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
+install(PROGRAMS  MCSInstanceCmds.sh MCSVolumeCmds.sh DESTINATION ${ENGINE_BINDIR} COMPONENT platform)

From 401c3b4929e6958dd63581f338f406493bcb7242 Mon Sep 17 00:00:00 2001
From: david hill <david.hill@mariadb.com>
Date: Wed, 9 Nov 2016 13:48:06 -0600
Subject: [PATCH 06/54] changed to use new MCS cloud scripts

---
 oam/oamcpp/liboamcpp.cpp | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp
index efd2cc977..49f8d369c 100644
--- a/oam/oamcpp/liboamcpp.cpp
+++ b/oam/oamcpp/liboamcpp.cpp
@@ -7567,7 +7567,7 @@ namespace oam
     std::string Oam::getEC2InstanceIpAddress(std::string instanceName)
 	{
 		// run script to get Instance status and IP Address
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh getPrivateIP " + instanceName + " > /tmp/getCloudIP_" + instanceName;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getPrivateIP " + instanceName + " > /tmp/getCloudIP_" + instanceName;
 		system(cmd.c_str());
 
 		if (checkLogStatus("/tmp/getCloudIP_" + instanceName, "stopped") )
@@ -7604,7 +7604,7 @@ namespace oam
     std::string Oam::getEC2LocalInstance(std::string name)
 	{
 		// run script to get Instance status and IP Address
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh getInstance  > /tmp/getInstanceInfo_" + name;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getInstance  > /tmp/getInstanceInfo_" + name;
 		int status = system(cmd.c_str());
 		if (WEXITSTATUS(status) != 0 )
 			return "failed";
@@ -7635,7 +7635,7 @@ namespace oam
     std::string Oam::getEC2LocalInstanceType(std::string name)
 	{
 		// run script to get Instance status and IP Address
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh getType  > /tmp/getInstanceType_" + name;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getType  > /tmp/getInstanceType_" + name;
 		int status = system(cmd.c_str());
 		if (WEXITSTATUS(status) != 0 )
 			return "failed";
@@ -7666,7 +7666,7 @@ namespace oam
     	std::string Oam::getEC2LocalInstanceSubnet(std::string name)
 	{
 		// run script to get Instance Subnet
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh getSubnet  > /tmp/getInstanceSubnet_" + name;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh getSubnet  > /tmp/getInstanceSubnet_" + name;
 		int status = system(cmd.c_str());
 		if (WEXITSTATUS(status) != 0 )
 			return "failed";
@@ -7698,7 +7698,7 @@ namespace oam
     std::string Oam::launchEC2Instance( const std::string name, const std::string IPAddress, const std::string type, const std::string group)
 	{
 		// run script to get Instance status and IP Address
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh launchInstance " + IPAddress + " " + type + " " + group + " > /tmp/getInstance_" + name;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh launchInstance " + IPAddress + " " + type + " " + group + " > /tmp/getInstance_" + name;
 		int status = system(cmd.c_str());
 		if (WEXITSTATUS(status) != 0 )
 			return "failed";
@@ -7740,7 +7740,7 @@ namespace oam
     void Oam::terminateEC2Instance(std::string instanceName)
 	{
 		// run script to get Instance status and IP Address
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh terminateInstance " + instanceName + " > /tmp/terminateEC2Instance_" + instanceName;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh terminateInstance " + instanceName + " > /tmp/terminateEC2Instance_" + instanceName;
 		system(cmd.c_str());
 
 		return;
@@ -7757,7 +7757,7 @@ namespace oam
     void Oam::stopEC2Instance(std::string instanceName)
 	{
 		// run script to get Instance status and IP Address
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh stopInstance " + instanceName + " > /tmp/stopEC2Instance_" + instanceName;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh stopInstance " + instanceName + " > /tmp/stopEC2Instance_" + instanceName;
 		system(cmd.c_str());
 
 		return;
@@ -7774,7 +7774,7 @@ namespace oam
     bool Oam::startEC2Instance(std::string instanceName)
 	{
 		// run script to get Instance status and IP Address
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh startInstance " + instanceName + " > /tmp/startEC2Instance_" + instanceName;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh startInstance " + instanceName + " > /tmp/startEC2Instance_" + instanceName;
 		int ret = system(cmd.c_str());
 		if (WEXITSTATUS(ret) != 0 )
 			return false;
@@ -7793,7 +7793,7 @@ namespace oam
     bool Oam::assignElasticIP(std::string instanceName, std::string IpAddress)
 	{
 		// run script to get Instance status and IP Address
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh assignElasticIP " + instanceName + " " + IpAddress + " > /tmp/assignElasticIP_" + instanceName;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh assignElasticIP " + instanceName + " " + IpAddress + " > /tmp/assignElasticIP_" + instanceName;
 		int ret = system(cmd.c_str());
 		if (WEXITSTATUS(ret) != 0 )
             exceptionControl("assignElasticIP", oam::API_FAILURE);
@@ -7812,7 +7812,7 @@ namespace oam
     bool Oam::deassignElasticIP(std::string IpAddress)
 	{
 		// run script to get Instance status and IP Address
-		string cmd = InstallDir + "/bin/IDBInstanceCmds.sh deassignElasticIP " + IpAddress + " > /tmp/deassignElasticIP_" + IpAddress;
+		string cmd = InstallDir + "/bin/MCSInstanceCmds.sh deassignElasticIP " + IpAddress + " > /tmp/deassignElasticIP_" + IpAddress;
 		int ret = system(cmd.c_str());
 		if (WEXITSTATUS(ret) != 0 )
             exceptionControl("deassignElasticIP", oam::API_FAILURE);
@@ -7831,7 +7831,7 @@ namespace oam
 	std::string Oam::getEC2VolumeStatus(std::string volumeName)
 	{
 		// run script to get Volume Status
-		string cmd = InstallDir + "/bin/IDBVolumeCmds.sh describe " + volumeName + " > /tmp/getVolumeStatus_" + volumeName;
+		string cmd = InstallDir + "/bin/MCSVolumeCmds.sh describe " + volumeName + " > /tmp/getVolumeStatus_" + volumeName;
 		int ret = system(cmd.c_str());
 		if (WEXITSTATUS(ret) != 0 )
 			return "failed";
@@ -7862,7 +7862,7 @@ namespace oam
 	std::string Oam::createEC2Volume(std::string size, std::string name)
 	{
 		// run script to get Volume Status
-		string cmd = InstallDir + "/bin/IDBVolumeCmds.sh create " + size + " " + name + " > /tmp/createVolumeStatus_" + name;
+		string cmd = InstallDir + "/bin/MCSVolumeCmds.sh create " + size + " " + name + " > /tmp/createVolumeStatus_" + name;
 		int ret = system(cmd.c_str());
 		if (WEXITSTATUS(ret) != 0 )
 			return "failed";
@@ -7906,7 +7906,7 @@ namespace oam
 		for ( int retry = 0 ; retry < 2 ; retry++ )
 		{
 			// run script to attach Volume
-			string cmd = InstallDir + "/bin/IDBVolumeCmds.sh attach " + volumeName + " " + instanceName + " " + deviceName + " > /tmp/attachVolumeStatus_" + volumeName;
+			string cmd = InstallDir + "/bin/MCSVolumeCmds.sh attach " + volumeName + " " + instanceName + " " + deviceName + " > /tmp/attachVolumeStatus_" + volumeName;
 			ret = system(cmd.c_str());
 
 			if (WEXITSTATUS(ret) == 0 )
@@ -7933,7 +7933,7 @@ namespace oam
 	bool Oam::detachEC2Volume(std::string volumeName)
 	{
 		// run script to attach Volume
-		string cmd = InstallDir + "/bin/IDBVolumeCmds.sh detach " + volumeName + " > /tmp/detachVolumeStatus_" + volumeName;
+		string cmd = InstallDir + "/bin/MCSVolumeCmds.sh detach " + volumeName + " > /tmp/detachVolumeStatus_" + volumeName;
 		int ret = system(cmd.c_str());
 		if (WEXITSTATUS(ret) != 0 )
 			return false;
@@ -7952,7 +7952,7 @@ namespace oam
 	bool Oam::deleteEC2Volume(std::string volumeName)
 	{
 		// run script to delete Volume
-		string cmd = InstallDir + "/bin/IDBVolumeCmds.sh delete " + volumeName + " > /tmp/deleteVolumeStatus_" + volumeName;
+		string cmd = InstallDir + "/bin/MCSVolumeCmds.sh delete " + volumeName + " > /tmp/deleteVolumeStatus_" + volumeName;
 		int ret = system(cmd.c_str());
 		if (WEXITSTATUS(ret) != 0 )
 			return false;
@@ -7971,7 +7971,7 @@ namespace oam
 	bool Oam::createEC2tag(std::string resourceName, std::string tagName, std::string tagValue)
 	{
 		// run script to create a tag
-		string cmd = InstallDir + "/bin/IDBVolumeCmds.sh createTag " + resourceName + " " + tagName + " " + tagValue + " > /tmp/createTagStatus_" + resourceName;
+		string cmd = InstallDir + "/bin/MCSVolumeCmds.sh createTag " + resourceName + " " + tagName + " " + tagValue + " > /tmp/createTagStatus_" + resourceName;
 		int ret = system(cmd.c_str());
 		if (WEXITSTATUS(ret) != 0 )
 			return false;

From 5450508c805e2c5573ed66a2fa5bb40a1f0fbda5 Mon Sep 17 00:00:00 2001
From: david hill <david.hill@mariadb.com>
Date: Wed, 9 Nov 2016 13:55:02 -0600
Subject: [PATCH 07/54] changed to use new MCS cloud scripts

---
 oam/cloud/MCSInstanceCmds.sh | 2 +-
 oam/cloud/MCSVolumeCmds.sh   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/oam/cloud/MCSInstanceCmds.sh b/oam/cloud/MCSInstanceCmds.sh
index 40d58f355..3e648abc3 100644
--- a/oam/cloud/MCSInstanceCmds.sh
+++ b/oam/cloud/MCSInstanceCmds.sh
@@ -1,5 +1,5 @@
 #! /bin/sh
-# IDBInstanceCmds.sh
+# MCSInstanceCmds.sh
 # get-local-instance-ID, get-zone, getPrivateIP from a Cloud environment
 #
 # 1. Amazon EC2
diff --git a/oam/cloud/MCSVolumeCmds.sh b/oam/cloud/MCSVolumeCmds.sh
index 3d64007b1..03fadfef8 100644
--- a/oam/cloud/MCSVolumeCmds.sh
+++ b/oam/cloud/MCSVolumeCmds.sh
@@ -1,5 +1,5 @@
 #! /bin/sh
-# IDBVolumeCmds.sh
+# MCSVolumeCmds.sh
 # describe, detach and attach Volume Storage from a Cloud environment
 #
 # 1. Amazon EC2

From 46011a75ee7ec59e4e295027dd5241919271f47c Mon Sep 17 00:00:00 2001
From: david hill <david.hill@mariadb.com>
Date: Wed, 9 Nov 2016 15:20:16 -0600
Subject: [PATCH 08/54] more changes for name change

---
 cpackEngineRPM.cmake       | 4 ++--
 oam/cloud/MCSVolumeCmds.sh | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake
index 7cee142a8..697572035 100644
--- a/cpackEngineRPM.cmake
+++ b/cpackEngineRPM.cmake
@@ -194,8 +194,8 @@ SET(CPACK_RPM_platform_USER_FILELIST
 "/usr/local/mariadb/columnstore/bin/autoConfigure"
 "/usr/local/mariadb/columnstore/bin/ddlcleanup"
 "/usr/local/mariadb/columnstore/bin/idbmeminfo"
-"/usr/local/mariadb/columnstore/bin/IDBInstanceCmds.sh"
-"/usr/local/mariadb/columnstore/bin/IDBVolumeCmds.sh"
+"/usr/local/mariadb/columnstore/bin/MCSInstanceCmds.sh"
+"/usr/local/mariadb/columnstore/bin/MCSVolumeCmds.sh"
 "/usr/local/mariadb/columnstore/bin/binary_installer.sh" 
 "/usr/local/mariadb/columnstore/bin/myCnf-include-args.text" 
 "/usr/local/mariadb/columnstore/bin/myCnf-exclude-args.text"
diff --git a/oam/cloud/MCSVolumeCmds.sh b/oam/cloud/MCSVolumeCmds.sh
index 03fadfef8..6c0e90290 100644
--- a/oam/cloud/MCSVolumeCmds.sh
+++ b/oam/cloud/MCSVolumeCmds.sh
@@ -189,7 +189,7 @@ checkInfostatus() {
 
 createvolume() {
 	# get zone
-	zone=`$prefix/mariadb/columnstore/bin/IDBInstanceCmds.sh getZone`
+	zone=`$prefix/mariadb/columnstore/bin/MCSinstanceCmds.sh getZone`
 
 	if [ $moduleType == "um" ]; then
 		# get type

From e9b41c569281465370c36976c06aab3b37944bc6 Mon Sep 17 00:00:00 2001
From: david hill <david.hill@mariadb.com>
Date: Wed, 9 Nov 2016 16:19:06 -0600
Subject: [PATCH 09/54] name change fix

---
 oam/cloud/MCSVolumeCmds.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oam/cloud/MCSVolumeCmds.sh b/oam/cloud/MCSVolumeCmds.sh
index 6c0e90290..1ee4dd024 100644
--- a/oam/cloud/MCSVolumeCmds.sh
+++ b/oam/cloud/MCSVolumeCmds.sh
@@ -189,7 +189,7 @@ checkInfostatus() {
 
 createvolume() {
 	# get zone
-	zone=`$prefix/mariadb/columnstore/bin/MCSinstanceCmds.sh getZone`
+	zone=`$prefix/mariadb/columnstore/bin/MCSInstanceCmds.sh getZone`
 
 	if [ $moduleType == "um" ]; then
 		# get type

From 5105b3281d2baa5174da2fc0f832aa8ef2f564ee Mon Sep 17 00:00:00 2001
From: Ben Thompson <dl3hp33ui@gmail.com>
Date: Wed, 9 Nov 2016 17:55:15 -0600
Subject: [PATCH 10/54] MCOL-387: Replaced option flags for version and release
 to not be tied to CPACK_RPM options. Fixed the defaults used in releasenum
 file.

---
 CMakeLists.txt       | 19 ++++++++++++++++---
 build/releasenum.in  |  4 ++--
 cpackEngineRPM.cmake |  4 ++--
 3 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 82a372f7b..edbe69e93 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -36,15 +36,28 @@ ENDIF(NOT CMAKE_BUILD_TYPE)
 
 #set( CMAKE_VERBOSE_MAKEFILE on )
 
+IF (NOT VERSION)
+    SET (VERSION 1.0.5)
+ENDIF()
+IF (NOT RELEASE)
+    SET (RELEASE 1)
+ENDIF()
+
 SET (PACKAGE columnstore)
-SET (VERSION 1.0.2)
 SET (PACKAGE_NAME columnstore)
 SET (PACKAGE_TARNAME columnstore)
-SET (PACKAGE_VERSION 1.0.2)
-SET (PACKAGE_STRING columnstore-1.0.2)
 SET (PACKAGE_BUGREPORT support@mariadb.com)
 SET (PACKAGE_URL "")
 
+IF (NOT PACKAGE_VERSION)
+    SET (PACKAGE_VERSION ${VERSION})
+ENDIF()
+IF (NOT PACKAGE_RELEASE)
+    SET (PACKAGE_RELEASE ${RELEASE})
+ENDIF()
+SET (PACKAGE_STRING columnstore-${PACKAGE_VERSION})
+
+
 SET (ENGINE_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 SET (INSTALL_ENGINE "/usr/local/mariadb/columnstore")
 
diff --git a/build/releasenum.in b/build/releasenum.in
index ea20eb5c1..1a05ad69d 100644
--- a/build/releasenum.in
+++ b/build/releasenum.in
@@ -1,2 +1,2 @@
-version=@CPACK_RPM_PACKAGE_VERSION@
-release=@CPACK_RPM_PACKAGE_RELEASE@
+version=@VERSION@
+release=@RELEASE@
\ No newline at end of file
diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake
index 7cee142a8..0290ea722 100644
--- a/cpackEngineRPM.cmake
+++ b/cpackEngineRPM.cmake
@@ -15,10 +15,10 @@ SET(CPACK_PACKAGE_NAME "mariadb-columnstore")
 SET(ENGINE_ARCH "x86_64")
 
 IF (NOT CPACK_RPM_PACKAGE_VERSION)
-SET (CPACK_RPM_PACKAGE_VERSION "1.0.0")
+SET (CPACK_RPM_PACKAGE_VERSION ${PACKAGE_VERSION})
 ENDIF()
 IF (NOT CPACK_RPM_PACKAGE_RELEASE)
-SET (CPACK_RPM_PACKAGE_RELEASE "0")
+SET (CPACK_RPM_PACKAGE_RELEASE ${PACKAGE_RELEASE})
 ENDIF()
 
 SET(CPACK_RPM_PACKAGE_NAME ${CPACK_PACKAGE_NAME})

From 16b47724346eed6e80a4edd6c88b5abb21f91e23 Mon Sep 17 00:00:00 2001
From: david hill <david.hill@mariadb.com>
Date: Thu, 10 Nov 2016 09:07:17 -0600
Subject: [PATCH 11/54] add sudo to chmod to non-root install

---
 oam/install_scripts/post-install | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install
index 38ad26fe6..3ca0e08e1 100755
--- a/oam/install_scripts/post-install
+++ b/oam/install_scripts/post-install
@@ -101,7 +101,7 @@ test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columns
 test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1
 test -h /var/log/mariadb/columnstore/data && rm -f /var/log/mariadb/columnstore/data
 # make sure trace dir is world-writable and sticky
-chmod 1755 /var/log/mariadb/columnstore/trace >/dev/null 2>&1
+sudo chmod 1755 /var/log/mariadb/columnstore/trace >/dev/null 2>&1
 test -d $installdir/data || mkdir $installdir/data
 test -d $installdir/data1 || mkdir $installdir/data1
 test -d $installdir/data1/systemFiles || mkdir $installdir/data1/systemFiles

From b2864a55c804b45772028cdb7de32eb1e01f1d42 Mon Sep 17 00:00:00 2001
From: david hill <david.hill@mariadb.com>
Date: Thu, 10 Nov 2016 09:24:11 -0600
Subject: [PATCH 12/54] remove sudo from ec2 check

---
 oamapps/postConfigure/postConfigure.cpp | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp
index c8353d478..e9bb20302 100644
--- a/oamapps/postConfigure/postConfigure.cpp
+++ b/oamapps/postConfigure/postConfigure.cpp
@@ -814,10 +814,7 @@ int main(int argc, char *argv[])
 
 	//amazon install setup check
 	bool amazonInstall = false;
-	if (rootUser)
-		system("ec2-version > /tmp/amazon.log 2>&1");
-	else
-		system("sudo ec2-version > /tmp/amazon.log 2>&1");
+	system("ec2-version > /tmp/amazon.log 2>&1");
 
 	ifstream in("/tmp/amazon.log");
 

From df65822c0d51cf346c18d606febc6669de9cce6a Mon Sep 17 00:00:00 2001
From: david hill <david.hill@mariadb.com>
Date: Thu, 10 Nov 2016 16:25:05 -0600
Subject: [PATCH 13/54] fix scp issue

---
 oamapps/postConfigure/postConfigure.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp
index e9bb20302..bee9dab69 100644
--- a/oamapps/postConfigure/postConfigure.cpp
+++ b/oamapps/postConfigure/postConfigure.cpp
@@ -3398,7 +3398,7 @@ int main(int argc, char *argv[])
 					//run remote command script
 					cout << endl << "----- Starting MariaDB Columnstore on '" + remoteModuleName + "' -----" << endl << endl;
 
-					cmd = installDir + "/bin/remote_scp_put.sh " + remoteModuleIP + " " + installDir + "/etc/Columnstore.xml " +  installDir + "/etc/. > /dev/null 2>&1";
+					cmd = installDir + "/bin/remote_scp_put.sh " + remoteModuleIP + " " + installDir + "/etc/Columnstore.xml  > /dev/null 2>&1";
 					int rtnCode = system(cmd.c_str());
 
 					cmd = installDir + "/bin/remote_command.sh " + remoteModuleIP + " " + password +

From 5ba2432a732c9f2860f9880557f792132d23a648 Mon Sep 17 00:00:00 2001
From: david hill <david.hill@mariadb.com>
Date: Thu, 10 Nov 2016 16:27:41 -0600
Subject: [PATCH 14/54] change prefix on all scripts for non-root install

---
 oam/install_scripts/post-install | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install
index 3ca0e08e1..b28f86a45 100755
--- a/oam/install_scripts/post-install
+++ b/oam/install_scripts/post-install
@@ -190,6 +190,9 @@ else
 
 	sed -i -e s@/usr/local/mariadb/columnstore@$installdir@g $installdir/bin/columnstore.def
 	sudo cp $installdir/bin/columnstore.def /etc/default/columnstore
+
+	sed -i -e s@prefix=/usr/local@prefix=$HOME@g $installdir/bin/*
+
 fi
 
 #check if MariaDB Columnstore system logging was setup

From e2b595c3e7a883dcd99f29788b48712bebe7bfd5 Mon Sep 17 00:00:00 2001
From: david hill <david.hill@mariadb.com>
Date: Fri, 11 Nov 2016 08:13:55 -0600
Subject: [PATCH 15/54] non-root changes

---
 oam/install_scripts/pre-uninstall | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall
index 464071188..ab85e38c0 100755
--- a/oam/install_scripts/pre-uninstall
+++ b/oam/install_scripts/pre-uninstall
@@ -65,23 +65,23 @@ fi
 #remove log file directories
 #rm -rf /var/log/mariadb/columnstore > /dev/null 2>&1
 #rm -f $installdir/mysql/db/*.err > /dev/null 2>&1
-rm -f /var/log/mariadb/columnstore/activeAlarms > /dev/null 2>&1
-rm -f /var/log/mariadb/columnstore/*.log1 > /dev/null 2>&1
+sudo rm -f /var/log/mariadb/columnstore/activeAlarms > /dev/null 2>&1
+sudo rm -f /var/log/mariadb/columnstore/*.log1 > /dev/null 2>&1
 rm -rf $installdir/mysql/db/columnstore_log_archive > /dev/null 2>&1
 sudo rm -f /etc/default/columnstore
 
 # remove Mariab Columnstore Log Rotate File abd Transaction Log File
-rm -f /etc/logrotate.d/columnstore > /dev/null 2>&1
-rm -f /etc/cron.d/transactionLog > /dev/null 2>&1
+sudo rm -f /etc/logrotate.d/columnstore > /dev/null 2>&1
+sudo rm -f /etc/cron.d/transactionLog > /dev/null 2>&1
 
 # delete Mariab Columnstore shared memory segments
 $installdir/bin/clearShm  > /dev/null 2>&1
 
 # delete prat files
-rm -f /etc/cron.d/ps > /dev/null 2>&1
-rm -f /etc/pscollect > /dev/null 2>&1
-/etc/init.d/crond reload  > /dev/null 2>&1
-sudo systemctl reload crond.service > /dev/null 2>&1
+sudo rm -f /etc/cron.d/ps > /dev/null 2>&1
+sudo rm -f /etc/pscollect > /dev/null 2>&1
+sudo /etc/init.d/crond reload  > /dev/null 2>&1
+sudo sudo systemctl reload crond.service > /dev/null 2>&1
 
 # delete tmp files
 rm -f $installdir/local/*.columnstore
@@ -90,7 +90,7 @@ rm -rf /tmp/bucketreuse
 rm -f /tmp/columnstore.txt
 rm -f /tmp/dbbuilder.*
 rm -f /tmp/dbrmfiles
-rm -f /var/lock/subsys/columnstore
+sudo rm -f /var/lock/subsys/columnstore
 rm -f /tmp/pkgcheck
 rm -f /tmp/upgrade-status.log.*
 rm -f /tmp/mount.log
@@ -122,8 +122,8 @@ systemctl=`which systemctl 2>/dev/null`
 if [ -n "$systemctl" ]; then
 
 	sudo systemctl disable columnstore >/dev/null 2>&1
-	rm -f /usr/lib/systemd/system/columnstore.service
-	rm -f /lib/systemd/system/columnstore.service
+	sudo rm -f /usr/lib/systemd/system/columnstore.service
+	sudo rm -f /lib/systemd/system/columnstore.service
 else
 	chkconfig=`which chkconfig 2>/dev/null`
 	if [ -n "$chkconfig" ]; then
@@ -158,9 +158,9 @@ fi
 #remove library paths
 if [ $installdir == "/usr/local/mariadb/columnstore" ]; then
         # remove library config file
-        rm -f /etc/ld.so.conf.d/columnstore.conf
-        rm -f /etc/ld.so.cache
-        ldconfig
+        sudo rm -f /etc/ld.so.conf.d/columnstore.conf
+        sudo rm -f /etc/ld.so.cache
+        sudo ldconfig
 fi
 
 #tell user to run post configure script

From 8da3f681a83e1d11ee4619e36fb958ee12685089 Mon Sep 17 00:00:00 2001
From: david hill <david.hill@mariadb.com>
Date: Fri, 11 Nov 2016 09:32:22 -0600
Subject: [PATCH 16/54] more non-root changes

---
 oam/install_scripts/columnstoreSyslog7 |  2 +-
 oam/install_scripts/post-install       | 19 +++----
 oam/install_scripts/syslogSetup.sh     | 75 +++++++++++++++-----------
 3 files changed, 56 insertions(+), 40 deletions(-)

diff --git a/oam/install_scripts/columnstoreSyslog7 b/oam/install_scripts/columnstoreSyslog7
index 87bababaa..0ed37f975 100644
--- a/oam/install_scripts/columnstoreSyslog7
+++ b/oam/install_scripts/columnstoreSyslog7
@@ -1,4 +1,4 @@
-# MariaDb Columnstore Database Platform Logging
+# MariaDB Columnstore Database Platform Logging
 local1.crit -/var/log/mariadb/columnstore/crit.log
 local1.err -/var/log/mariadb/columnstore/err.log
 local1.warning -/var/log/mariadb/columnstore/warning.log
diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install
index b28f86a45..10655b4bc 100755
--- a/oam/install_scripts/post-install
+++ b/oam/install_scripts/post-install
@@ -8,9 +8,12 @@ prefix=/usr/local
 installdir=$prefix/mariadb/columnstore
 rpmmode=install
 user=$USER
+sudo=sudo
 if [ -z "$user" ]; then
-	user=root
+        user=root
+        sudo=" "
 fi
+
 quiet=0
 
 for arg in "$@"; do
@@ -95,13 +98,11 @@ sudo ln -s /usr/lib64/perl5/CORE/libperl.so /usr/lib64/libperl.so >/dev/null 2>&
 
 cd /
 
-test -d /var/log/mariadb/columnstore || mkdir /var/log/mariadb/columnstore >/dev/null 2>&1
-test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1
-test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1
-test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1
-test -h /var/log/mariadb/columnstore/data && rm -f /var/log/mariadb/columnstore/data
+test -d /var/log/mariadb/columnstore || sudo mkdir /var/log/mariadb/columnstore >/dev/null 2>&1
+test -d /var/log/mariadb/columnstore/archive || sudo mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1
+test -d /var/log/mariadb/columnstore/corefiles || sudo mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1
+test -h /var/log/mariadb/columnstore/data && sudo rm -f /var/log/mariadb/columnstore/data
 # make sure trace dir is world-writable and sticky
-sudo chmod 1755 /var/log/mariadb/columnstore/trace >/dev/null 2>&1
 test -d $installdir/data || mkdir $installdir/data
 test -d $installdir/data1 || mkdir $installdir/data1
 test -d $installdir/data1/systemFiles || mkdir $installdir/data1/systemFiles
@@ -180,10 +181,10 @@ if [ $user = "root" ]; then
 	chmod 777 -R /dev/shm
 	rm -f /etc/default/columnstore
 else
+        sudo chmod 755 /var/log
+        sudo chmod -R 755 /var/log/mariadb/columnstore
 	sudo $installdir/bin/syslogSetup.sh --installdir=$installdir install > /tmp/syslog_install.log 2>&1
 	sudo chown $user:$user $installdir/etc/Columnstore.xml
-	sudo chmod 755 /var/log
-	sudo chmod -R 755 /var/log/mariadb/columnstore
 	sudo chmod -R 777 /dev/shm
 	sudo rm -f /var/lock/subsys/mysql-Columnstore
 	sudo rm -f /tmp/StopColumnstore
diff --git a/oam/install_scripts/syslogSetup.sh b/oam/install_scripts/syslogSetup.sh
index 73315d1b6..ae606f3d4 100755
--- a/oam/install_scripts/syslogSetup.sh
+++ b/oam/install_scripts/syslogSetup.sh
@@ -5,13 +5,21 @@
 # syslogSetup.sh - install / uninstall MariaDB Columnstore system logging configuration
 
 # no point in going any further if not root... (only works in bash)
-test $EUID -eq 0 || exit 0
+#test $EUID -eq 0 || exit 0
 
 prefix=/usr/local
 installdir=$prefix/mariadb/columnstore
 syslog_conf=nofile
 rsyslog7=0
 
+user=$USER
+sudo=sudo
+if [ -z "$user" ]; then
+        user=root
+        sudo=" "
+fi
+
+
 for arg in "$@"; do
 	if [ `expr -- "$arg" : '--prefix='` -eq 9 ]; then
 		prefix="`echo $arg | awk -F= '{print $2}'`"
@@ -62,13 +70,13 @@ if [ "$daemon" = "nodaemon" ]; then
 
 	if [ -f /etc/syslog.conf ]; then
 		daemon="syslog"
-		/etc/init.d/syslog start > /dev/null 2>&1
+		sudo /etc/init.d/syslog start > /dev/null 2>&1
 	elif [ -f /etc/rsyslog.conf ]; then
 		daemon="rsyslog"
-		/etc/init.d/rsyslog start > /dev/null 2>&1
+		sudo /etc/init.d/rsyslog start > /dev/null 2>&1
 	elif [ -f /etc/init.d/syslog-ng ]; then
 		daemon="syslog-ng"
-		/etc/init.d/syslog-ng start > /dev/null 2>&1
+		sudo /etc/init.d/syslog-ng start > /dev/null 2>&1
 	fi
 fi
 
@@ -92,7 +100,7 @@ if [ "$daemon" = "syslog-ng" ]; then
 	fi
 elif [ "$daemon" = "rsyslog" ]; then
 	#check if rsyslog version 7 or greater
-	rsyslogd -v > /tmp/rsyslog.ver
+	sudo rsyslogd -v > /tmp/rsyslog.ver
 	cnt=`grep "rsyslogd 7" /tmp/rsyslog.ver | wc -l`
 	if [ $cnt -gt 0 ]; then
 		rsyslog7=1
@@ -149,30 +157,33 @@ checkSyslog
 if [ ! -z "$syslog_conf" ] ; then
 	$installdir/bin/setConfig -d Installation SystemLogConfigFile ${syslog_conf} >/dev/null 2>&1
 	if [ "$syslog_conf" != /etc/rsyslog.d/columnstore.conf ]; then
-		rm -f ${syslog_conf}.columnstoreSave
-		cp ${syslog_conf} ${syslog_conf}.columnstoreSave >/dev/null 2>&1
-		sed -i '/# MariaDB/,$d' ${syslog_conf}.columnstoreSave > /dev/null 2>&1
+		sudo rm -f ${syslog_conf}.columnstoreSave
+		sudo cp ${syslog_conf} ${syslog_conf}.columnstoreSave >/dev/null 2>&1
+		sudo sed -i '/# MariaDB/,$d' ${syslog_conf}.columnstoreSave > /dev/null 2>&1
 	fi
 
 	egrep -qs 'MariaDB Columnstore Database Platform Logging' ${syslog_conf}
 	if [ $? -ne 0 ]; then
-		#set the syslog for calpont logging
+		#set the syslog for ColumnStore logging
 		# remove older version incase it was installed by previous build
-		rm -rf /etc/rsyslog.d/columnstore.conf
+		sudo rm -rf /etc/rsyslog.d/columnstore.conf
 		if [ $rsyslog7 == 1 ]; then
-			rm -f /etc/rsyslog.d/49-columnstore.conf
-			cat  ${columnstoreSyslogFile7} >> ${syslog_conf}
-			chown syslog:adm /var/log/mariadb/columnstore
+			sudo rm -f /etc/rsyslog.d/49-columnstore.conf
+			sudo cp  ${columnstoreSyslogFile7} ${syslog_conf}
+			sudo chown syslog:adm /var/log/mariadb/columnstore >/dev/null 2>&1
 		else
-			cat  ${columnstoreSyslogFile} >> ${syslog_conf}
+			sudo cp  ${columnstoreSyslogFile} ${syslog_conf}
 		fi
 	fi
 
-	pkill -hup syslogd > /dev/null 2>&1
-	pkill -hup syslog-ng  > /dev/null 2>&1
-	pkill -hup rsyslogd  > /dev/null 2>&1
-	/etc/init.d/rsyslog restart  > /dev/null 2>&1
+	sudo etc/init.d/rsyslog restart  > /dev/null 2>&1
+	sudo /etc/init.d/syslog restart  > /dev/null 2>&1
+        sudo /etc/init.d/syslog-ng restart  > /dev/null 2>&1
+
 	systemctl restart rsyslog.service > /dev/null 2>&1
+        systemctl restart syslog.service > /dev/null 2>&1
+        systemctl restart syslog-ng.service > /dev/null 2>&1
+
 fi
 
 }
@@ -185,26 +196,30 @@ if [ ! -z "$syslog_conf" ] ; then
 			egrep -qs 'MariaDB Columnstore Database Platform Logging' ${syslog_conf}
 			if [ $? -eq 0 ]; then
 				if [ -f ${syslog_conf}.columnstoreSave ] ; then
-					#uninstall the syslog for calpont logging
-					mv -f ${syslog_conf} ${syslog_conf}.calpontBackup
-					mv -f ${syslog_conf}.columnstoreSave ${syslog_conf} >/dev/null 2>&1
+					#uninstall the syslog for ColumnStore logging
+					sudo v -f ${syslog_conf} ${syslog_conf}.ColumnStoreBackup
+					sudo mv -f ${syslog_conf}.columnstoreSave ${syslog_conf} >/dev/null 2>&1
 					if [ ! -f ${syslog_conf} ] ; then
-						cp ${syslog_conf}.calpontBackup ${syslog_conf}
+						sudo cp ${syslog_conf}.ColumnStoreBackup ${syslog_conf}
 					fi
 				fi
 			fi
-			sed -i '/# MariaDB/,$d' ${syslog_conf} > /dev/null 2>&1
+			sudo sed -i '/# MariaDB/,$d' ${syslog_conf} > /dev/null 2>&1
 		else
-			rm -f "$syslog_conf"
+			sudo rm -f "$syslog_conf"
 		fi
 	else
-		rm -f "$syslog_conf"
+		sudo rm -f "$syslog_conf"
 	fi
 
-	pkill -hup syslogd > /dev/null 2>&1
-	pkill -hup syslog-ng  > /dev/null 2>&1
-	/etc/init.d/rsyslog restart  > /dev/null 2>&1
-	systemctl restart rsyslog.service > /dev/null 2>&1
+        sudo etc/init.d/rsyslog restart  > /dev/null 2>&1
+        sudo /etc/init.d/syslog restart  > /dev/null 2>&1
+        sudo /etc/init.d/syslog-ng restart  > /dev/null 2>&1
+
+        systemctl restart rsyslog.service > /dev/null 2>&1
+        systemctl restart syslog.service > /dev/null 2>&1
+        systemctl restart syslog-ng.service > /dev/null 2>&1
+
 
 	$installdir/bin/setConfig -d Installation SystemLogConfigFile "unassigned"
 
@@ -227,7 +242,7 @@ fi
 check() {
 test -f $installdir/post/functions && . $installdir/post/functions
 number=$RANDOM
-cplogger -i 104 "MariaDB Columnstore Log Test: $number"
+$installdir/bin/cplogger -i 104 "MariaDB Columnstore Log Test: $number"
 sleep 3
 egrep -qs "MariaDB Columnstore Log Test: $number" /var/log/mariadb/columnstore/info.log
 if [ $? -eq 0 ]; then

From 5aced3442f3c75e19ae573c4c2fd6314dbf670a3 Mon Sep 17 00:00:00 2001
From: david hill <david.hill@mariadb.com>
Date: Fri, 11 Nov 2016 14:09:00 -0600
Subject: [PATCH 17/54] mcol-398

---
 oamapps/postConfigure/installer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp
index 418830649..cc567ce55 100644
--- a/oamapps/postConfigure/installer.cpp
+++ b/oamapps/postConfigure/installer.cpp
@@ -289,7 +289,7 @@ int main(int argc, char *argv[])
 							value = "8G";
 					}
 
-					cout << "      Setting 'TotalUmMemory' to " << percent << "% of total memory (Combined Server Install maximum value is 16G). Value set to " << value << endl;
+					cout << "      Setting 'TotalUmMemory' to " << percent << "% of total memory. Value set to " << value << endl;
 	
 					try {
 						sysConfig->setConfig("HashJoin", "TotalUmMemory", value);

From 874596c6cd1050137406c38de968673ae1952824 Mon Sep 17 00:00:00 2001
From: Andrew Hutchings <andrew@linuxjedi.co.uk>
Date: Mon, 14 Nov 2016 15:41:25 +0000
Subject: [PATCH 18/54] MCOL-309 Implement Information Schema tables

This patch does the following:

* Fix a year storage issue in the datestamp for the columnstore tables
  metadata table (note that any previous tables will still have the
  incorrect year stored).
* Expose the table creation date in CalpontSystemCatalog::getTables()
* Add an INFORMATION_SCHEMA table listing the tables in ColumnStore
  (similar to systable)
* Add an INFORMATION_SCHEMA table listing the columns in ColumnStore
  (similar to syscolumn)
* Add an INFORMATION_SCHEMA table listing the extents in ColumnStore
  (similar to the editem tool but with additional file information)
* Modifies the build system and scripts to support the new tables
---
 cpackEngineRPM.cmake                     |   9 +
 dbcon/execplan/calpontsystemcatalog.cpp  |  11 ++
 dbcon/execplan/calpontsystemcatalog.h    |   1 +
 dbcon/mysql/CMakeLists.txt               |  37 +++-
 dbcon/mysql/install_calpont_mysql.sh     |   3 +
 dbcon/mysql/is_columnstore_columns.cpp   | 187 +++++++++++++++++++
 dbcon/mysql/is_columnstore_extents.cpp   | 222 +++++++++++++++++++++++
 dbcon/mysql/is_columnstore_tables.cpp    | 113 ++++++++++++
 oam/install_scripts/post-mysqld-install  |  11 +-
 writeengine/server/we_ddlcommandproc.cpp |   2 +-
 10 files changed, 592 insertions(+), 4 deletions(-)
 create mode 100644 dbcon/mysql/is_columnstore_columns.cpp
 create mode 100644 dbcon/mysql/is_columnstore_extents.cpp
 create mode 100644 dbcon/mysql/is_columnstore_tables.cpp

diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake
index 0290ea722..0a222978e 100644
--- a/cpackEngineRPM.cmake
+++ b/cpackEngineRPM.cmake
@@ -318,6 +318,15 @@ SET(CPACK_RPM_storage-engine_USER_FILELIST
 "/usr/local/mariadb/columnstore/lib/libudf_mysql.so.1.0.0"
 "/usr/local/mariadb/columnstore/lib/libudf_mysql.so.1"
 "/usr/local/mariadb/columnstore/lib/libudf_mysql.so"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_columns.so"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_columns.so.1"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_columns.so.1.0.0"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_extents.so"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_extents.so.1"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_extents.so.1.0.0"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_tables.so"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_tables.so.1"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_tables.so.1.0.0"
 "/usr/local/mariadb/columnstore/mysql/mysql-Columnstore"
 "/usr/local/mariadb/columnstore/mysql/install_calpont_mysql.sh"
 "/usr/local/mariadb/columnstore/mysql/syscatalog_mysql.sql"
diff --git a/dbcon/execplan/calpontsystemcatalog.cpp b/dbcon/execplan/calpontsystemcatalog.cpp
index ecec4fcf9..635927c7b 100644
--- a/dbcon/execplan/calpontsystemcatalog.cpp
+++ b/dbcon/execplan/calpontsystemcatalog.cpp
@@ -2599,6 +2599,7 @@ const vector< pair<CalpontSystemCatalog::OID, CalpontSystemCatalog::TableName> >
     SimpleColumn *c1 = new SimpleColumn(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+TABLENAME_COL, fSessionID);
     SimpleColumn *c2 = new SimpleColumn(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+SCHEMA_COL, fSessionID);
     SimpleColumn *c3 = new SimpleColumn(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+OBJECTID_COL, fSessionID);
+    SimpleColumn *c4 = new SimpleColumn(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+CREATEDATE_COL, fSessionID);
     
     SRCP srcp;
     srcp.reset(c1);
@@ -2607,6 +2608,8 @@ const vector< pair<CalpontSystemCatalog::OID, CalpontSystemCatalog::TableName> >
     colMap.insert(CMVT_(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+SCHEMA_COL, srcp));
     srcp.reset(c3);
     colMap.insert(CMVT_(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+OBJECTID_COL, srcp));
+    srcp.reset(c4);
+    colMap.insert(CMVT_(CALPONT_SCHEMA+"."+SYSTABLE_TABLE+"."+CREATEDATE_COL, srcp));
     csep.columnMapNonStatic(colMap);
         
     srcp.reset(c1->clone());
@@ -2653,6 +2656,14 @@ const vector< pair<CalpontSystemCatalog::OID, CalpontSystemCatalog::TableName> >
           tables[i].second.schema = (*it)->GetStringData(i);
       }
     }
+    for (it = sysDataList.begin(); it != sysDataList.end(); it++)
+    {
+        if ((*it)->ColumnOID() == c4->oid())
+        {
+            for (int i = 0; i < (*it)->dataCount(); i++)
+                tables[i].second.create_date = (*it)->GetData(i);
+        }
+    }
     
     for (it = sysDataList.begin(); it != sysDataList.end(); it++)
     {
diff --git a/dbcon/execplan/calpontsystemcatalog.h b/dbcon/execplan/calpontsystemcatalog.h
index 42b721d65..2936b2daa 100644
--- a/dbcon/execplan/calpontsystemcatalog.h
+++ b/dbcon/execplan/calpontsystemcatalog.h
@@ -431,6 +431,7 @@ public:
 		TableName(const TableAliasName& tan): schema(tan.schema), table(tan.table) {}
 		std::string schema;
 		std::string table;
+        int64_t create_date;
 		bool operator<(const TableName& rhs) const;
 		bool operator>=(const TableName& rhs) const { return !(*this < rhs); }
 		bool operator==(const TableName& rhs) const
diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt
index 7dde5b208..7d9b012ca 100644
--- a/dbcon/mysql/CMakeLists.txt
+++ b/dbcon/mysql/CMakeLists.txt
@@ -17,7 +17,7 @@ SET ( libcalmysql_SRCS
     ha_view.cpp sm.cpp
     ha_window_function.cpp
     ha_calpont_partition.cpp
-    ha_pseudocolumn.cpp )
+    ha_pseudocolumn.cpp)
 
 add_definitions(-DMYSQL_DYNAMIC_PLUGIN)
 
@@ -29,7 +29,40 @@ target_link_libraries(calmysql ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} ${NETSNMP_
 
 set_target_properties(calmysql PROPERTIES VERSION 1.0.0 SOVERSION 1)
 
-install(TARGETS calmysql DESTINATION ${ENGINE_LIBDIR} COMPONENT storage-engine)
+SET ( is_columnstore_tables_SRCS
+    is_columnstore_tables.cpp
+    )
+add_library(is_columnstore_tables SHARED ${is_columnstore_tables_SRCS})
+
+target_link_libraries(is_columnstore_tables ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool)
+
+# Don't prepend .so file with 'lib'
+set_target_properties(is_columnstore_tables PROPERTIES PREFIX "")
+set_target_properties(is_columnstore_tables PROPERTIES VERSION 1.0.0 SOVERSION 1)
+
+SET ( is_columnstore_columns_SRCS
+    is_columnstore_columns.cpp
+    )
+add_library(is_columnstore_columns SHARED ${is_columnstore_columns_SRCS})
+
+target_link_libraries(is_columnstore_columns ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool)
+
+# Don't prepend .so file with 'lib'
+set_target_properties(is_columnstore_columns PROPERTIES PREFIX "")
+set_target_properties(is_columnstore_columns PROPERTIES VERSION 1.0.0 SOVERSION 1)
+
+SET ( is_columnstore_extents_SRCS
+    is_columnstore_extents.cpp
+    )
+add_library(is_columnstore_extents SHARED ${is_columnstore_extents_SRCS})
+
+target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool)
+
+# Don't prepend .so file with 'lib'
+set_target_properties(is_columnstore_extents PROPERTIES PREFIX "")
+set_target_properties(is_columnstore_extents PROPERTIES VERSION 1.0.0 SOVERSION 1)
+
+install(TARGETS calmysql is_columnstore_tables is_columnstore_columns is_columnstore_extents DESTINATION ${ENGINE_LIBDIR} COMPONENT storage-engine)
 install(FILES syscatalog_mysql.sql 
               dumpcat_mysql.sql 
               calsetuserpriority.sql 
diff --git a/dbcon/mysql/install_calpont_mysql.sh b/dbcon/mysql/install_calpont_mysql.sh
index 5b2b981df..d04bc3487 100755
--- a/dbcon/mysql/install_calpont_mysql.sh
+++ b/dbcon/mysql/install_calpont_mysql.sh
@@ -30,6 +30,9 @@ df=$installdir/mysql/my.cnf
 $installdir/mysql/bin/mysql --defaults-file=$df --force --user=root $pwprompt mysql 2>/tmp/mysql_install.log <<EOD
 INSTALL PLUGIN columnstore SONAME 'libcalmysql.so';
 INSTALL PLUGIN infinidb SONAME 'libcalmysql.so';
+INSTALL PLUGIN columnstore_tables SONAME 'is_columnstore_tables.so';
+INSTALL PLUGIN columnstore_columns SONAME 'is_columnstore_columns.so';
+INSTALL PLUGIN columnstore_extents SONAME 'is_columnstore_extents.so';
 -- these are deprecated names
 DELETE FROM mysql.func WHERE name='caldisablepartition';
 DELETE FROM mysql.func WHERE name='caldroppartition';
diff --git a/dbcon/mysql/is_columnstore_columns.cpp b/dbcon/mysql/is_columnstore_columns.cpp
new file mode 100644
index 000000000..d97ca8823
--- /dev/null
+++ b/dbcon/mysql/is_columnstore_columns.cpp
@@ -0,0 +1,187 @@
+/* c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil
+ * vi: set shiftwidth=4 tabstop=4 expandtab:
+ *  :indentSize=4:tabSize=4:noTabs=true:
+ *
+ * Copyright (C) 2016 MariaDB Corporaton
+ *
+ * This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2 of
+ * the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "idb_mysql.h"
+#include <vector>
+#include <limits>
+
+#include <boost/shared_ptr.hpp>
+#include "calpontsystemcatalog.h"
+#include "dataconvert.h"
+
+
+// Required declaration as it isn't in a MairaDB include
+bool schema_table_store_record(THD *thd, TABLE *table);
+
+ST_FIELD_INFO is_columnstore_columns_fields[] =
+{
+    {"TABLE_SCHEMA", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},
+    {"TABLE_NAME", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},
+    {"COLUMN_NAME", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},
+    {"OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"DICTIONARY_OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, MY_I_S_MAYBE_NULL, 0, 0},
+    {"LIST_OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, MY_I_S_MAYBE_NULL, 0, 0},
+    {"TREE_OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, MY_I_S_MAYBE_NULL, 0, 0},
+    {"DATA_TYPE", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},
+    {"COLUMN_LENGTH", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"COLUMN_POSITION", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"COLUMN_DEFAULT", 0, MYSQL_TYPE_LONG_BLOB, 0, MY_I_S_MAYBE_NULL, 0, 0},
+    {"IS_NULLABLE", 1, MYSQL_TYPE_TINY, 0, 0, 0, 0},
+    {"NUMERIC_PRECISION", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"NUMERIC_SCALE", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"IS_AUTOINCREMENT", 1, MYSQL_TYPE_TINY, 0, 0, 0, 0},
+    {"COMPRESSION_TYPE", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},
+    {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0}
+
+};
+
+static int is_columnstore_columns_fill(THD *thd, TABLE_LIST *tables, COND *cond)
+{
+    CHARSET_INFO *cs = system_charset_info;
+    TABLE *table = tables->table;
+
+    boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr =
+        execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(0);
+
+    const std::vector< std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> > catalog_tables
+        = systemCatalogPtr->getTables();
+
+    for (std::vector<std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> >::const_iterator it = catalog_tables.begin();
+         it != catalog_tables.end(); ++it)
+    {
+        execplan::CalpontSystemCatalog::RIDList column_rid_list = systemCatalogPtr->columnRIDs((*it).second);
+        for (size_t col_num = 0; col_num < column_rid_list.size(); col_num++)
+        {
+            execplan::CalpontSystemCatalog::TableColName tcn = systemCatalogPtr->colName(column_rid_list[col_num].objnum);
+            execplan::CalpontSystemCatalog::ColType ct = systemCatalogPtr->colType(column_rid_list[col_num].objnum);
+
+			table->field[0]->store(tcn.schema.c_str(), tcn.schema.length(), cs);
+			table->field[1]->store(tcn.table.c_str(), tcn.table.length(), cs);
+			table->field[2]->store(tcn.column.c_str(), tcn.column.length(), cs);
+			table->field[3]->store(column_rid_list[col_num].objnum);
+			if (ct.ddn.dictOID == std::numeric_limits<int32_t>::min())
+			{
+			    table->field[4]->set_null();
+			}
+			else
+			{
+			    table->field[4]->set_notnull();
+			    table->field[4]->store(ct.ddn.dictOID);
+			}
+			if (ct.ddn.listOID == std::numeric_limits<int32_t>::min())
+			{
+			    table->field[5]->set_null();
+			}
+			else
+			{
+			    table->field[5]->set_notnull();
+			    table->field[5]->store(ct.ddn.listOID);
+			}
+			if (ct.ddn.treeOID == std::numeric_limits<int32_t>::min())
+			{
+			    table->field[6]->set_null();
+			}
+			else
+			{
+			    table->field[6]->set_notnull();
+			    table->field[6]->store(ct.ddn.treeOID);
+			}
+			std::string data_type = execplan::colDataTypeToString(ct.colDataType);
+			table->field[7]->store(data_type.c_str(), data_type.length(), cs);
+			table->field[8]->store(ct.colWidth);
+			table->field[9]->store(ct.colPosition);
+			if (ct.defaultValue.empty())
+			{
+			    table->field[10]->set_null();
+			}
+			else
+			{
+			    table->field[10]->set_notnull();
+    			table->field[10]->store(ct.defaultValue.c_str(), ct.defaultValue.length(), cs);
+    		}
+    		table->field[11]->store(ct.autoincrement);
+			table->field[12]->store(ct.precision);
+			table->field[13]->store(ct.scale);
+			if (ct.constraintType != execplan::CalpontSystemCatalog::NOTNULL_CONSTRAINT)
+			{
+			    table->field[14]->store(true);
+			}
+			else
+			{
+			    table->field[14]->store(false);
+			}
+
+			std::string compression_type;
+			switch (ct.compressionType)
+			{
+				case 0:
+				    compression_type = "None";
+				    break;
+				case 2:
+				    compression_type = "Snappy";
+				    break;
+				default:
+				    compression_type = "Unknown";
+				    break;
+			}
+			table->field[15]->store(compression_type.c_str(), compression_type.length(), cs);
+            if (schema_table_store_record(thd, table))
+                return 1;
+        }
+    }
+
+
+
+    return 0;
+}
+
+static int is_columnstore_columns_plugin_init(void *p)
+{
+    ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE*) p;
+    schema->fields_info = is_columnstore_columns_fields;
+    schema->fill_table = is_columnstore_columns_fill;
+    return 0;
+}
+
+static struct st_mysql_information_schema is_columnstore_columns_plugin_version =
+{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION };
+
+maria_declare_plugin(is_columnstore_columns_plugin)
+{
+    MYSQL_INFORMATION_SCHEMA_PLUGIN,
+    &is_columnstore_columns_plugin_version,
+    "COLUMNSTORE_COLUMNS",
+    "MariaDB Corporaton",
+    "An information schema plugin to list ColumnStore columns",
+    PLUGIN_LICENSE_GPL,
+    is_columnstore_columns_plugin_init,
+    //is_columnstore_tables_plugin_deinit,
+    NULL,
+    0x0100,
+    NULL,
+    NULL,
+    "1.0",
+    MariaDB_PLUGIN_MATURITY_STABLE
+}
+maria_declare_plugin_end;
+
+
diff --git a/dbcon/mysql/is_columnstore_extents.cpp b/dbcon/mysql/is_columnstore_extents.cpp
new file mode 100644
index 000000000..fa5835888
--- /dev/null
+++ b/dbcon/mysql/is_columnstore_extents.cpp
@@ -0,0 +1,222 @@
+/* c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil
+ * vi: set shiftwidth=4 tabstop=4 expandtab:
+ *  :indentSize=4:tabSize=4:noTabs=true:
+ *
+ * Copyright (C) 2016 MariaDB Corporaton
+ *
+ * This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2 of
+ * the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "idb_mysql.h"
+#include <vector>
+#include <limits>
+
+#include <boost/shared_ptr.hpp>
+#include "dbrm.h"
+#include "objectidmanager.h"
+#include "we_convertor.h"
+#include "we_define.h"
+#include "IDBPolicy.h"
+#include "we_config.h"
+
+// Required declaration as it isn't in a MairaDB include
+bool schema_table_store_record(THD *thd, TABLE *table);
+
+ST_FIELD_INFO is_columnstore_extents_fields[] =
+{
+    {"OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},                     // 0
+    {"OBJECT_TYPE", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},                 // 1
+    {"LOGICAL_BLOCK_START", 19, MYSQL_TYPE_LONGLONG, 0, 0, 0, 0},       // 2
+    {"LOGICAL_BLOCK_END", 19, MYSQL_TYPE_LONGLONG, 0, 0, 0, 0},         // 3
+    {"MIN_VALUE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, // 4
+    {"MAX_VALUE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, // 5
+    {"WIDTH", 1, MYSQL_TYPE_TINY, 0, 0, 0, 0},                          // 6
+    {"DBROOT", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},                        // 7
+    {"PARTITION_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},                  // 8
+    {"SEGMENT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},                    // 9
+    {"BLOCK_OFFSET", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},                  // 10
+    {"MAX_BLOCKS", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},                    // 11
+    {"HIGH_WATER_MARK", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},               // 12
+    {"STATE", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},                       // 13
+    {"STATUS", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},                      // 14
+    {"DATA_SIZE", 19, MYSQL_TYPE_LONGLONG, 0, 0, 0, 0},                 // 15
+    {"FILENAME", 1024, MYSQL_TYPE_STRING, 0, 0, 0, 0},                  // 16
+    {"FILE_SIZE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, // 17
+    {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0}
+};
+
+static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond)
+{
+    CHARSET_INFO *cs = system_charset_info;
+    TABLE *table = tables->table;
+    std::vector<struct BRM::EMEntry> entries;
+   	std::vector<struct BRM::EMEntry>::iterator iter;
+	std::vector<struct BRM::EMEntry>::iterator end;
+	char oidDirName[WriteEngine::FILE_NAME_SIZE];
+	char fullFileName[WriteEngine::FILE_NAME_SIZE];
+    char dbDir[WriteEngine::MAX_DB_DIR_LEVEL][WriteEngine::MAX_DB_DIR_NAME_SIZE];
+
+    BRM::DBRM *emp = new BRM::DBRM();
+    if (!emp || !emp->isDBRMReady())
+    {
+        return 1;
+    }
+
+    execplan::ObjectIDManager oidm;
+    BRM::OID_t MaxOID = oidm.size();
+
+    for(BRM::OID_t oid = 0; oid <= MaxOID; oid++)
+    {
+        emp->getExtents(oid, entries, false, false, true);
+        if (entries.size() == 0)
+            continue;
+
+        iter = entries.begin();
+        end = entries.end();
+        while (iter != end)
+        {
+            table->field[0]->store(oid);
+            if (iter->colWid > 0)
+            {
+                table->field[1]->store("Column", strlen("Column"), cs);
+                if (iter->partition.cprange.lo_val == std::numeric_limits<int64_t>::max() ||
+        			iter->partition.cprange.lo_val <= (std::numeric_limits<int64_t>::min() + 2))
+        		{
+        		    table->field[4]->set_null();
+        		}
+        		else
+        		{
+        		    table->field[4]->set_notnull();
+                    table->field[4]->store(iter->partition.cprange.lo_val);
+                }
+                if (iter->partition.cprange.hi_val == std::numeric_limits<int64_t>::max() ||
+        			iter->partition.cprange.hi_val <= (std::numeric_limits<int64_t>::min() + 2))
+        		{
+        		    table->field[5]->set_null();
+        		}
+        		else
+        		{
+                    table->field[5]->set_notnull();
+                    table->field[5]->store(iter->partition.cprange.hi_val);
+                }
+                table->field[6]->store(iter->colWid);
+
+            }
+            else
+            {
+                table->field[1]->store("Dictionary", strlen("Dictionary"), cs);
+                table->field[4]->set_null();
+                table->field[5]->set_null();
+                table->field[6]->store(8192);
+            }
+            table->field[2]->store(iter->range.start);
+            table->field[3]->store(iter->range.start + (iter->range.size * 1024) - 1);
+
+            table->field[7]->store(iter->dbRoot);
+            table->field[8]->store(iter->partitionNum);
+            table->field[9]->store(iter->segmentNum);
+            table->field[10]->store(iter->blockOffset);
+            table->field[11]->store(iter->range.size * 1024);
+            table->field[12]->store(iter->HWM);
+
+            switch (iter->partition.cprange.isValid)
+            {
+                case 0:
+                    table->field[13]->store("Invalid", strlen("Invalid"), cs);
+                    break;
+                case 1:
+                    table->field[13]->store("Updating", strlen("Updating"), cs);
+                    break;
+                case 2:
+                    table->field[13]->store("Valid", strlen("Valid"), cs);
+                    break;
+                default:
+                    table->field[13]->store("Unknown", strlen("Unknown"), cs);
+                    break;
+            }
+            switch (iter->status)
+            {
+                case BRM::EXTENTAVAILABLE:
+                    table->field[14]->store("Available", strlen("Available"), cs);
+                    break;
+                case BRM::EXTENTUNAVAILABLE:
+                    table->field[14]->store("Unavailable", strlen("Unavailable"), cs);
+                    break;
+                case BRM::EXTENTOUTOFSERVICE:
+                    table->field[14]->store("Out of service", strlen("Out of service"), cs);
+                    break;
+                default:
+                    table->field[14]->store("Unknown", strlen("Unknown"), cs);
+            }
+            table->field[15]->store((iter->HWM + 1) * 8192);
+
+            WriteEngine::Convertor::oid2FileName(oid, oidDirName, dbDir, iter->partitionNum, iter->segmentNum);
+            snprintf(fullFileName, WriteEngine::FILE_NAME_SIZE, "%s/%s", WriteEngine::Config::getDBRootByNum(iter->dbRoot).c_str(), oidDirName);
+            table->field[16]->store(fullFileName, strlen(fullFileName), cs);
+
+            if (idbdatafile::IDBPolicy::exists(fullFileName))
+            {
+                table->field[17]->set_notnull();
+                table->field[17]->store(idbdatafile::IDBPolicy::size(fullFileName));
+            }
+            else
+            {
+                table->field[17]->set_null();
+            }
+
+            if (schema_table_store_record(thd, table))
+                return 1;
+
+            iter++;
+
+        }
+    }
+
+
+    return 0;
+}
+
+static int is_columnstore_extents_plugin_init(void *p)
+{
+    ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE*) p;
+    schema->fields_info = is_columnstore_extents_fields;
+    schema->fill_table = is_columnstore_extents_fill;
+    return 0;
+}
+
+static struct st_mysql_information_schema is_columnstore_extents_plugin_version =
+{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION };
+
+maria_declare_plugin(is_columnstore_extents_plugin)
+{
+    MYSQL_INFORMATION_SCHEMA_PLUGIN,
+    &is_columnstore_extents_plugin_version,
+    "COLUMNSTORE_EXTENTS",
+    "MariaDB Corporaton",
+    "An information schema plugin to list ColumnStore extents",
+    PLUGIN_LICENSE_GPL,
+    is_columnstore_extents_plugin_init,
+    //is_columnstore_extents_plugin_deinit,
+    NULL,
+    0x0100,
+    NULL,
+    NULL,
+    "1.0",
+    MariaDB_PLUGIN_MATURITY_STABLE
+}
+maria_declare_plugin_end;
+
+
diff --git a/dbcon/mysql/is_columnstore_tables.cpp b/dbcon/mysql/is_columnstore_tables.cpp
new file mode 100644
index 000000000..e38c8d5e7
--- /dev/null
+++ b/dbcon/mysql/is_columnstore_tables.cpp
@@ -0,0 +1,113 @@
+/* c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil
+ * vi: set shiftwidth=4 tabstop=4 expandtab:
+ *  :indentSize=4:tabSize=4:noTabs=true:
+ *
+ * Copyright (C) 2016 MariaDB Corporaton
+ *
+ * This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2 of
+ * the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "idb_mysql.h"
+#include <vector>
+
+#include <boost/shared_ptr.hpp>
+#include "calpontsystemcatalog.h"
+#include "dataconvert.h"
+
+
+// Required declaration as it isn't in a MairaDB include
+bool schema_table_store_record(THD *thd, TABLE *table);
+
+ST_FIELD_INFO is_columnstore_tables_fields[] =
+{
+    {"TABLE_SCHEMA", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},
+    {"TABLE_NAME", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},
+    {"OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"CREATION_DATE", 0, MYSQL_TYPE_DATE, 0, 0, 0, 0},
+    {"COLUMN_COUNT", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"AUTOINCREMENT", 11, MYSQL_TYPE_LONG, 0, MY_I_S_MAYBE_NULL, 0, 0},
+    {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0}
+};
+
+static int is_columnstore_tables_fill(THD *thd, TABLE_LIST *tables, COND *cond)
+{
+    CHARSET_INFO *cs = system_charset_info;
+    TABLE *table = tables->table;
+
+    boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr =
+        execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(0);
+
+    const std::vector< std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> > catalog_tables
+        = systemCatalogPtr->getTables();
+
+    for (std::vector<std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> >::const_iterator it = catalog_tables.begin();
+         it != catalog_tables.end(); ++it)
+    {
+        execplan::CalpontSystemCatalog::TableInfo tb_info = systemCatalogPtr->tableInfo((*it).second);
+        std::string create_date = dataconvert::DataConvert::dateToString((*it).second.create_date);
+        table->field[0]->store((*it).second.schema.c_str(), (*it).second.schema.length(), cs);
+        table->field[1]->store((*it).second.table.c_str(), (*it).second.table.length(), cs);
+        table->field[2]->store((*it).first);
+        table->field[3]->store(create_date.c_str(), create_date.length(), cs);
+        table->field[4]->store(tb_info.numOfCols);
+        if (tb_info.tablewithautoincr)
+        {
+            table->field[5]->set_notnull();
+            table->field[5]->store(systemCatalogPtr->nextAutoIncrValue((*it).second));
+        }
+        else
+        {
+            table->field[5]->set_null();
+        }
+        table->field[5]->store(tb_info.tablewithautoincr);
+        if (schema_table_store_record(thd, table))
+            return 1;
+    }
+
+    return 0;
+}
+
+static int is_columnstore_tables_plugin_init(void *p)
+{
+    ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE*) p;
+    schema->fields_info = is_columnstore_tables_fields;
+    schema->fill_table = is_columnstore_tables_fill;
+    return 0;
+}
+
+static struct st_mysql_information_schema is_columnstore_tables_plugin_version =
+{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION };
+
+maria_declare_plugin(is_columnstore_tables_plugin)
+{
+    MYSQL_INFORMATION_SCHEMA_PLUGIN,
+    &is_columnstore_tables_plugin_version,
+    "COLUMNSTORE_TABLES",
+    "MariaDB Corporaton",
+    "An information schema plugin to list ColumnStore tables",
+    PLUGIN_LICENSE_GPL,
+    is_columnstore_tables_plugin_init,
+    //is_columnstore_tables_plugin_deinit,
+    NULL,
+    0x0100,
+    NULL,
+    NULL,
+    "1.0",
+    MariaDB_PLUGIN_MATURITY_STABLE
+}
+maria_declare_plugin_end;
+
+
diff --git a/oam/install_scripts/post-mysqld-install b/oam/install_scripts/post-mysqld-install
index 833f5f26c..92ea035fa 100755
--- a/oam/install_scripts/post-mysqld-install
+++ b/oam/install_scripts/post-mysqld-install
@@ -82,9 +82,15 @@ chown -R $user.$user $installdir/mysql
 if [ -f $installdir/lib/libcalmysql.so.1.0.0 ]; then
 	libcalmysql=$installdir/lib/libcalmysql.so.1.0.0
 	libudfsdk=$installdir/lib/libudf_mysql.so.1.0.0
+    is_columnstore_tables=$installdir/lib/is_columnstore_tables.so.1.0.0
+    is_columnstore_columns=$installdir/lib/is_columnstore_columns.so.1.0.0
+    is_columnstore_extents=$installdir/lib/is_columnstore_extents.so.1.0.0
 elif [ -f $installdir/lib/libcalmysql.so.1 ]; then
 	libcalmysql=$installdir/lib/libcalmysql.so.1
 	libudfsdk=$installdir/lib/libudf_mysql.so.1
+    is_columnstore_tables=$installdir/lib/is_columnstore_tables.so.1
+    is_columnstore_columns=$installdir/lib/is_columnstore_columns.so.1
+    is_columnstore_extents=$installdir/lib/is_columnstore_extents.so.1
 else
 	libcalmysql=
 fi
@@ -93,8 +99,11 @@ if [ -n "$libcalmysql" ]; then
 
 	cd $installdir/mysql/lib/plugin
 	ln -sf $libcalmysql libcalmysql.so
-        ln -sf $libcalmysql libcalmysqlent.so
+    ln -sf $libcalmysql libcalmysqlent.so
 	ln -sf $libudfsdk libudf_mysql.so
+    ln -sf $is_columnstore_tables is_columnstore_tables.so
+    ln -sf $is_columnstore_columns is_columnstore_columns.so
+    ln -sf $is_columnstore_extents is_columnstore_extents.so
 fi
 
 # cleanup previous install mysql replication files
diff --git a/writeengine/server/we_ddlcommandproc.cpp b/writeengine/server/we_ddlcommandproc.cpp
index b896b25a3..2ccdd08ae 100644
--- a/writeengine/server/we_ddlcommandproc.cpp
+++ b/writeengine/server/we_ddlcommandproc.cpp
@@ -193,7 +193,7 @@ uint8_t WE_DDLCommandProc::writeSystable(ByteStream& bs, std::string &err)
 
                 t = time(NULL);
                 gmtime_r(&t, &tmp);
-				aDay.year = tmp.tm_year;
+				aDay.year = tmp.tm_year + 1900;
 				aDay.month = tmp.tm_mon+1;
 				aDay.day = tmp.tm_mday;
 

From 469256815aba9384b82c6fb06962380f80c3de5d Mon Sep 17 00:00:00 2001
From: David Hall <david.hall@mariadb.com>
Date: Mon, 14 Nov 2016 17:54:03 -0600
Subject: [PATCH 19/54] MCOL-311 Add findObjectFile command to mcsadmin

---
 oam/etc/ConsoleCmds.xml         |  7 ++-
 oamapps/mcsadmin/CMakeLists.txt |  2 +-
 oamapps/mcsadmin/mcsadmin.cpp   | 93 ++++++++++++++++++++++++++++++++-
 3 files changed, 98 insertions(+), 4 deletions(-)

diff --git a/oam/etc/ConsoleCmds.xml b/oam/etc/ConsoleCmds.xml
index 01b258751..a5a29e750 100644
--- a/oam/etc/ConsoleCmds.xml
+++ b/oam/etc/ConsoleCmds.xml
@@ -29,7 +29,12 @@
                 <Name>AVAILABLE</Name>
 	</Cmd4>
 	<Cmd5>
-                <Name>AVAILABLE</Name>
+		<Name>findObjectFile</Name>
+		<Desc1>Get the name of the directory containing the first file of the object</Desc1>
+		<Arg1>Object ID (OID) of object</Arg1>
+		<Arg2>Schema</Arg2>
+		<Arg3>Table</Arg3>
+		<Arg4>Column</Arg4>
 	</Cmd5>
 	<Cmd6>
 		<Name>getModuleTypeConfig</Name>
diff --git a/oamapps/mcsadmin/CMakeLists.txt b/oamapps/mcsadmin/CMakeLists.txt
index 5d06594df..c8d4af0e2 100644
--- a/oamapps/mcsadmin/CMakeLists.txt
+++ b/oamapps/mcsadmin/CMakeLists.txt
@@ -8,7 +8,7 @@ set(mcsadmin_SRCS mcsadmin.cpp)
 
 add_executable(mcsadmin ${mcsadmin_SRCS})
 
-target_link_libraries(mcsadmin ${ENGINE_LDFLAGS} readline ncurses  ${ENGINE_EXEC_LIBS})
+target_link_libraries(mcsadmin ${ENGINE_LDFLAGS} readline ncurses  ${ENGINE_EXEC_LIBS} ${ENGINE_WRITE_LIBS})
 
 install(TARGETS mcsadmin DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
 
diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp
index 50af96f0b..f15e760d3 100644
--- a/oamapps/mcsadmin/mcsadmin.cpp
+++ b/oamapps/mcsadmin/mcsadmin.cpp
@@ -32,13 +32,15 @@ extern int h_errno;
 #include "boost/tokenizer.hpp"
 #include "sessionmanager.h"
 #include "dbrm.h"
+#include "we_config.h" // for findObjectFile
+#include "we_fileop.h" // for findObjectFile
 namespace fs = boost::filesystem;
 
 using namespace alarmmanager;
 using namespace std;
 using namespace oam;
 using namespace config;
-
+using namespace execplan;
 #include "installdir.h"
 
 // Variables shared in both main and functions
@@ -707,8 +709,95 @@ int processCommand(string* arguments)
         }
         break;
 
-        case 5: // Available
+        case 5: // findObjectFile
         {
+			unsigned maxDBRoot = WriteEngine::Config::DBRootCount();
+			if (maxDBRoot < 1)
+			{
+				cout << endl << "getDatafileName fails because there are no dbroots defined for this server" << endl;
+				break;;
+			}
+			if (arguments[1] == "")
+			{
+				cout << endl << "getDatafileName requires one of" << endl;
+				cout << "a) oid of column for which file name is to be retrieved" << endl;
+				cout << "b) schema, table and column for which file name is to be retrieved" << endl;
+				break;
+			}
+			char* endchar;
+			int oid = strtol(arguments[1].c_str(), &endchar, 0);
+			// test to see if not all numeric
+			if (endchar < &(*arguments[1].end()))
+			{
+				oid = 0;
+			}
+			if (oid == 0)
+			{
+				// Need to convert the arguments to oid
+				boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr =
+					execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(0);
+				CalpontSystemCatalog::TableColName columnName;
+				columnName.schema = arguments[1];
+				if (arguments[2] == "")
+				{
+					cout << endl << "getDatafileName requires a table and column for schema " << arguments[1] << endl;
+					break;
+				}
+				columnName.table = arguments[2];
+				if (arguments[3] == "")
+				{
+					// No column was given. Use the first column in the table.
+					CalpontSystemCatalog::TableName tableName;
+					tableName.schema = arguments[1];
+					tableName.table = arguments[2];
+					CalpontSystemCatalog::RIDList rdlist = systemCatalogPtr->columnRIDs(tableName);
+					oid = rdlist.front().objnum;
+				}
+				else
+				{
+					columnName.column = arguments[3];
+					oid = systemCatalogPtr->lookupOID(columnName);
+				}
+			}
+
+			// Use writeengine code to get the filename
+			WriteEngine::FileOp fileOp;
+			char fileName[WriteEngine::FILE_NAME_SIZE];
+			memset(fileName, 0, WriteEngine::FILE_NAME_SIZE);
+			int rc;
+
+			if (oid < 1000)
+				rc = fileOp.getVBFileName(oid, fileName);
+			else
+				rc = fileOp.oid2DirName(oid, fileName);
+			cout << "file name for oid " << oid << ":" << endl;
+			if (strlen(fileName) > 0)
+			{
+				cout << fileName;
+			}
+			if (rc == WriteEngine::NO_ERROR)
+			{
+				// Success. No more output.
+				cout << endl;
+			}
+			else if (rc == WriteEngine::ERR_FILE_NOT_EXIST)
+			{
+				if (strlen(fileName) == 0)
+				{
+					// We couldn't get a name
+					cout << "Error: Filename could not be determined" << endl;
+				}
+				else
+				{
+					// We got a name, but the file doesn't exist
+					cout << " (OID directory not found)" << endl;
+				}
+			}
+			else
+			{
+				// Something broke
+				cerr << "WriteEngine::FileOp::oid2DirName() error. rc=" << rc << endl;
+			}
         }
         break;
 

From 3b33601a2075774c4fec7ccf53eab97debc13246 Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Tue, 15 Nov 2016 16:30:58 +0000
Subject: [PATCH 20/54] mcol-406, fix non-root startup issue

---
 oam/oamcpp/liboamcpp.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp
index 49f8d369c..b7d60b1e6 100644
--- a/oam/oamcpp/liboamcpp.cpp
+++ b/oam/oamcpp/liboamcpp.cpp
@@ -9615,7 +9615,7 @@ namespace oam
 			// The stat above may fail for non-root because of permissions
 			// This is a non-optimal solution
 			string cmd = "pgrep ProcMon";
-			if (system(cmd.c_str()) == 0)
+			if (system(cmd.c_str()) == 1)
 			{
 				return true;
 			}

From cc0f5efa28bbdeb7b67c427d39995da5382fd89e Mon Sep 17 00:00:00 2001
From: Ben Thompson <dl3hp33ui@gmail.com>
Date: Tue, 15 Nov 2016 11:18:27 -0600
Subject: [PATCH 21/54] MCOL-386: Added some error reporting in catch
 statements in helpers.cpp

---
 oamapps/postConfigure/helpers.cpp | 93 +++++++++++++++++++++----------
 1 file changed, 63 insertions(+), 30 deletions(-)

diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp
index dd3ff7e2e..36cca5177 100644
--- a/oamapps/postConfigure/helpers.cpp
+++ b/oamapps/postConfigure/helpers.cpp
@@ -114,8 +114,10 @@ void dbrmDirCheck()
 		dbrmroot = sysConfig->getConfig(SystemSection, "DBRMRoot");
 		dbrmrootPrev = sysConfigPrev->getConfig(SystemSection, "DBRMRoot");
 	}
-	catch(...)
-	{}
+	catch(const std::exception &exc)
+	{
+	    std::cerr << exc.what() << std::endl;
+	}
 
 	if ( dbrmrootPrev.empty() )
 		return;
@@ -259,8 +261,10 @@ void mysqlSetup()
 	try {
 		oam.actionMysqlCalpont(MYSQL_START);
 	}
-	catch(...)
-	{}
+	catch(const std::exception &exc)
+	{
+	    std::cerr << exc.what() << std::endl;
+	}
 	sleep(2);
 	
 	for (;;)
@@ -289,7 +293,10 @@ void mysqlSetup()
 			try {
 				oam.setSystemConfig("MySQLPasswordConfig", "y");
 			}
-			catch(...) {}
+			catch(const std::exception &exc)
+			{
+			    std::cerr << exc.what() << std::endl;
+			}
 		}
 		else
 		{
@@ -303,8 +310,10 @@ void mysqlSetup()
 				try {
 					oam.actionMysqlCalpont(MYSQL_STOP);
 				}
-				catch(...)
-				{}
+				catch(const std::exception &exc)
+				{
+				    std::cerr << exc.what() << std::endl;
+				}
 				unlink("/tmp/idbmysql.log");
 				break;
 			}
@@ -344,8 +353,10 @@ int sendUpgradeRequest(int IserverTypeInstall, bool pmwithum)
 			if ( procstat.ProcessOpState == oam::ACTIVE)
 				break;
 		}
-		catch (exception& ex)
-		{}
+		catch(const std::exception &exc)
+		{
+		    std::cerr << exc.what() << std::endl;
+		}
 	}
 
 	SystemModuleTypeConfig systemmoduletypeconfig;
@@ -353,8 +364,10 @@ int sendUpgradeRequest(int IserverTypeInstall, bool pmwithum)
 	try{
 		oam.getSystemConfig(systemmoduletypeconfig);
 	}
-	catch (exception& ex)
-	{}
+	catch(const std::exception &exc)
+	{
+	    std::cerr << exc.what() << std::endl;
+	}
 
 	ByteStream msg;
 	ByteStream::byte requestID = RUNUPGRADE;
@@ -393,8 +406,10 @@ int sendUpgradeRequest(int IserverTypeInstall, bool pmwithum)
 						}
 					}
 				}
-				catch (exception& ex)
-				{}
+				catch(const std::exception &exc)
+				{
+				    std::cerr << exc.what() << std::endl;
+				}
 			}
 		}
 	}
@@ -417,8 +432,10 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st
 	try{
 		oam.getSystemConfig(systemmoduletypeconfig);
 	}
-	catch (exception& ex)
-	{}
+	catch(const std::exception &exc)
+	{
+	    std::cerr << exc.what() << std::endl;
+	}
 
 	//get Primary (Master) UM
 	string masterModule = oam::UnassignedName;
@@ -530,8 +547,10 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st
 						pt++;
 					}
 				}
-				catch (exception& ex)
-				{}
+				catch(const std::exception &exc)
+				{
+				    std::cerr << exc.what() << std::endl;
+				}
 			}
 			else
 				pt++;
@@ -593,7 +612,9 @@ int sendMsgProcMon( std::string module, ByteStream msg, int requestID, int timeo
 				try {
 					receivedMSG = mqRequest.read(&ts);
 				}
-				catch (...) {
+				catch (const std::exception &exc)
+				{
+					std::cerr << exc.what() << std::endl;
 					return returnStatus;
 				}
 	
@@ -630,8 +651,10 @@ int sendMsgProcMon( std::string module, ByteStream msg, int requestID, int timeo
 
 		mqRequest.shutdown();
 	}
-	catch (exception& ex)
-	{}
+	catch(const std::exception &exc)
+	{
+	    std::cerr << exc.what() << std::endl;
+	}
 
 	return returnStatus;
 }
@@ -651,8 +674,10 @@ void checkFilesPerPartion(int DBRootCount, Config* sysConfig)
 	try {
 		dbRoot = sysConfig->getConfig(SystemSection, "DBRoot1");
 	}
-	catch(...)
-	{}
+	catch(const std::exception &exc)
+	{
+	    std::cerr << exc.what() << std::endl;
+	}
 
 	dbRoot = dbRoot + "/000.dir";
 
@@ -661,8 +686,10 @@ void checkFilesPerPartion(int DBRootCount, Config* sysConfig)
 		string tmp = sysConfig->getConfig("ExtentMap", "FilesPerColumnPartition");
 		FilesPerColumnPartition = atoi(tmp.c_str());
 	}
-	catch(...)
-	{}
+	catch(const std::exception &exc)
+	{
+	    std::cerr << exc.what() << std::endl;
+	}
 
 	if ( fmod(FilesPerColumnPartition , (float) DBRootCount) != 0 ) {
 		ifstream oldFile (dbRoot.c_str());
@@ -738,8 +765,10 @@ void checkMysqlPort( std::string& mysqlPort, Config* sysConfig )
 				try {
 					sysConfig->setConfig("Installation", "MySQLPort", mysqlPort);
 				}
-				catch(...)
-				{}
+				catch(const std::exception &exc)
+				{
+				    std::cerr << exc.what() << std::endl;
+				}
 		
 				if ( !writeConfig(sysConfig) ) {
 					cout << "ERROR: Failed trying to update MariDB Columnstore System Configuration file" << endl;
@@ -860,8 +889,10 @@ void checkSystemMySQLPort(std::string& mysqlPort, Config* sysConfig, std::string
 			try {
 				sysConfig->setConfig("Installation", "MySQLPort", mysqlPort);
 			}
-			catch(...)
-			{}
+			catch(const std::exception &exc)
+			{
+			    std::cerr << exc.what() << std::endl;
+			}
 	
 			if ( !writeConfig(sysConfig) ) {
 				cout << "ERROR: Failed trying to update MariDB Columnstore System Configuration file" << endl;
@@ -889,8 +920,10 @@ bool writeConfig( Config* sysConfig )
 			sysConfig->write();
 			return true;
 		}
-		catch(...)
-		{}
+		catch(const std::exception &exc)
+		{
+		    std::cerr << exc.what() << std::endl;
+		}
 	}
 
 	return false;

From a74b03fdad9443f8e95c1f44d166a10d23d90e21 Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Wed, 16 Nov 2016 23:22:25 +0000
Subject: [PATCH 22/54] mcol-404, non-root install fixes

---
 oam/install_scripts/columnstore         | 20 ++++++-----
 oam/install_scripts/post-install        | 13 ++++++--
 oam/install_scripts/pre-uninstall       | 44 +++++++++++++++----------
 oam/oamcpp/liboamcpp.cpp                |  2 +-
 oamapps/alarmmanager/alarmmanager.cpp   | 20 +++--------
 oamapps/postConfigure/postConfigure.cpp | 11 +++----
 6 files changed, 58 insertions(+), 52 deletions(-)

diff --git a/oam/install_scripts/columnstore b/oam/install_scripts/columnstore
index c346e3eb0..315256cd9 100644
--- a/oam/install_scripts/columnstore
+++ b/oam/install_scripts/columnstore
@@ -57,8 +57,10 @@ if [ "x$has_um" = x ]; then
 fi
 
 user=$USER
+sudo=sudo
 if [ -z "$user" ]; then
         user=root
+        sudo=" "
 fi
 
 checkInstallSetup() {
@@ -66,7 +68,7 @@ checkInstallSetup() {
 	if [ $InitialInstallFlag != "y" ]; then
 		echo "Please run the postConfigure install script, check the Installation Guide"
 		echo "for additional details"
-		rm -f /var/lock/subsys/columnstore
+		sudo rm -f /var/lock/subsys/columnstore
 		exit 1
 	fi
 }
@@ -79,13 +81,13 @@ start() {
 		exit 0
 	fi
 
-	(mkdir -p /var/lock/subsys && touch /var/lock/subsys/columnstore) >/dev/null 2>&1
+	(sudo mkdir -p /var/lock/subsys && touch /var/lock/subsys/columnstore) >/dev/null 2>&1
 
 	if [ -x $InstallDir/bin/columnstore.pre-start ]; then
 		$InstallDir/bin/columnstore.pre-start
 		if [ $? -ne 0 ]; then
 			echo "Error running MariaDB Columnstore pre-start script, not starting MariaDB Columnstore"
-			rm -f /var/lock/subsys/columnstore
+			sudo rm -f /var/lock/subsys/columnstore
 			exit 1
 		fi
 	fi
@@ -106,21 +108,21 @@ start() {
 
 	RETVAL=0
 	echo "Starting MariaDB Columnstore Database Platform"
-	rm -f /tmp/StopColumnstore
+	sudo rm -f /tmp/StopColumnstore
 	exec $InstallDir/bin/run.sh $InstallDir/bin/ProcMon  > /dev/null 2>&1 &
 
 	return $RETVAL
 }	
 stop() {
 	echo "Shutting down MariaDB Columnstore Database Platform"
-	touch /tmp/StopColumnstore
-	pkill ProcMon
-	pkill ProcMgr
+	sudo touch /tmp/StopColumnstore
+	sudo pkill ProcMon
+	sudo pkill ProcMgr
 	sleep 1
 	$InstallDir/bin/clearShm
 	RETVAL=$?
-	rm -f /var/lock/subsys/columnstore
-	fuser -k 8604/tcp > /dev/null 2>&1
+	sudo rm -f /var/lock/subsys/columnstore
+	sudo fuser -k 8604/tcp > /dev/null 2>&1
 	test -f $InstallDir/mysql/mysql-Columnstore || return $RETVAL
 	$InstallDir/mysql/mysql-Columnstore stop > /dev/null 2>&1
 	if [ -x $InstallDir/bin/columnstore.post-stop ]; then
diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install
index 38ad26fe6..6e245b31b 100755
--- a/oam/install_scripts/post-install
+++ b/oam/install_scripts/post-install
@@ -95,7 +95,16 @@ sudo ln -s /usr/lib64/perl5/CORE/libperl.so /usr/lib64/libperl.so >/dev/null 2>&
 
 cd /
 
+if [ $user != "root" ]; then
+	sudo chmod 777 /var
+	sudo chmod 777 /var/log
+fi
+
+test -d /var/log/mariadb || mkdir /var/log/mariadb >/dev/null 2>&1
+sudo chmod 777 /var/log/mariadb
 test -d /var/log/mariadb/columnstore || mkdir /var/log/mariadb/columnstore >/dev/null 2>&1
+sudo chmod 777 /var/log/mariadb/columnstore
+
 test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1
 test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1
 test -d /var/log/mariadb/columnstore/trace || mkdir /var/log/mariadb/columnstore/trace >/dev/null 2>&1
@@ -182,9 +191,9 @@ if [ $user = "root" ]; then
 else
 	sudo $installdir/bin/syslogSetup.sh --installdir=$installdir install > /tmp/syslog_install.log 2>&1
 	sudo chown $user:$user $installdir/etc/Columnstore.xml
-	sudo chmod 755 /var/log
-	sudo chmod -R 755 /var/log/mariadb/columnstore
 	sudo chmod -R 777 /dev/shm
+	sudo chmod 777 /var/lock/subsys
+	sudo chmod 777 /tmp
 	sudo rm -f /var/lock/subsys/mysql-Columnstore
 	sudo rm -f /tmp/StopColumnstore
 
diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall
index 464071188..bf3bbc053 100755
--- a/oam/install_scripts/pre-uninstall
+++ b/oam/install_scripts/pre-uninstall
@@ -38,17 +38,27 @@ $installdir/bin/columnstore stop > /dev/null 2>&1
 $installdir/myql/columnstore-Mysql stop > /dev/null 2>&1
 
 # uninstall OS scripts updated by postConfigure
-if test -f /root/.bashrc.columnstoreSave ; then
-	$sudo mv -f /root/.bashrc.columnstoreSave /root/.bashrc > /dev/null 2>&1
+if [ $user = "root" ]; then
+	if test -f /root/.bashrc.columnstoreSave ; then
+		mv -f /root/.bashrc.columnstoreSave /root/.bashrc > /dev/null 2>&1
+	fi
+else
+        if test -f $HOME/.bashrc.columnstoreSave ; then
+                mv -f /home/$user/.bashrc.columnstoreSave /home/$user/.bashrc > /dev/null 2>&1
+        fi
 fi
+	
+
 if test -f /etc/exports.columnstoreSave ; then
 	$sudo mv -f /etc/exports.columnstoreSave /etc/exports > /dev/null 2>&1
 else
 	$sudo rm -f /etc/exports > /dev/null 2>&1
 fi
-if test -f /etc/rc.d/rc.local.columnstoreSave ; then
+
+if test -f /etc/rc.d/rc.local.columnstoreSave ; then	
 	$sudo mv -f /etc/rc.d/rc.local.columnstoreSave /etc/rc.d/rc.local > /dev/null 2>&1
 fi
+
 if test -f /etc/rc.local.columnstoreSave ; then
 	$sudo mv -f /etc/rc.local.columnstoreSave /etc/rc.local > /dev/null 2>&1
 fi
@@ -65,23 +75,23 @@ fi
 #remove log file directories
 #rm -rf /var/log/mariadb/columnstore > /dev/null 2>&1
 #rm -f $installdir/mysql/db/*.err > /dev/null 2>&1
-rm -f /var/log/mariadb/columnstore/activeAlarms > /dev/null 2>&1
-rm -f /var/log/mariadb/columnstore/*.log1 > /dev/null 2>&1
+sudo rm -f /var/log/mariadb/columnstore/activeAlarms > /dev/null 2>&1
+sudo rm -f /var/log/mariadb/columnstore/*.log1 > /dev/null 2>&1
 rm -rf $installdir/mysql/db/columnstore_log_archive > /dev/null 2>&1
 sudo rm -f /etc/default/columnstore
 
 # remove Mariab Columnstore Log Rotate File abd Transaction Log File
-rm -f /etc/logrotate.d/columnstore > /dev/null 2>&1
-rm -f /etc/cron.d/transactionLog > /dev/null 2>&1
+sudo rm -f /etc/logrotate.d/columnstore > /dev/null 2>&1
+sudo rm -f /etc/cron.d/transactionLog > /dev/null 2>&1
 
 # delete Mariab Columnstore shared memory segments
 $installdir/bin/clearShm  > /dev/null 2>&1
 
 # delete prat files
-rm -f /etc/cron.d/ps > /dev/null 2>&1
-rm -f /etc/pscollect > /dev/null 2>&1
-/etc/init.d/crond reload  > /dev/null 2>&1
-sudo systemctl reload crond.service > /dev/null 2>&1
+sudo rm -f /etc/cron.d/ps > /dev/null 2>&1
+sudo rm -f /etc/pscollect > /dev/null 2>&1
+sudo /etc/init.d/crond reload  > /dev/null 2>&1
+sudo sudo systemctl reload crond.service > /dev/null 2>&1
 
 # delete tmp files
 rm -f $installdir/local/*.columnstore
@@ -90,7 +100,7 @@ rm -rf /tmp/bucketreuse
 rm -f /tmp/columnstore.txt
 rm -f /tmp/dbbuilder.*
 rm -f /tmp/dbrmfiles
-rm -f /var/lock/subsys/columnstore
+sudo rm -f /var/lock/subsys/columnstore
 rm -f /tmp/pkgcheck
 rm -f /tmp/upgrade-status.log.*
 rm -f /tmp/mount.log
@@ -122,8 +132,8 @@ systemctl=`which systemctl 2>/dev/null`
 if [ -n "$systemctl" ]; then
 
 	sudo systemctl disable columnstore >/dev/null 2>&1
-	rm -f /usr/lib/systemd/system/columnstore.service
-	rm -f /lib/systemd/system/columnstore.service
+	sudo rm -f /usr/lib/systemd/system/columnstore.service
+	sudo rm -f /lib/systemd/system/columnstore.service
 else
 	chkconfig=`which chkconfig 2>/dev/null`
 	if [ -n "$chkconfig" ]; then
@@ -158,9 +168,9 @@ fi
 #remove library paths
 if [ $installdir == "/usr/local/mariadb/columnstore" ]; then
         # remove library config file
-        rm -f /etc/ld.so.conf.d/columnstore.conf
-        rm -f /etc/ld.so.cache
-        ldconfig
+        sudo rm -f /etc/ld.so.conf.d/columnstore.conf
+        sudo rm -f /etc/ld.so.cache
+        sudo ldconfig
 fi
 
 #tell user to run post configure script
diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp
index efd2cc977..55dbdefea 100644
--- a/oam/oamcpp/liboamcpp.cpp
+++ b/oam/oamcpp/liboamcpp.cpp
@@ -9614,7 +9614,7 @@ namespace oam
 			// not root user
 			// The stat above may fail for non-root because of permissions
 			// This is a non-optimal solution
-			string cmd = "pgrep ProcMon";
+			string cmd = "pgrep ProcMon > /dev/null 2>&1";
 			if (system(cmd.c_str()) == 0)
 			{
 				return true;
diff --git a/oamapps/alarmmanager/alarmmanager.cpp b/oamapps/alarmmanager/alarmmanager.cpp
index 499cc7755..7d33b0302 100644
--- a/oamapps/alarmmanager/alarmmanager.cpp
+++ b/oamapps/alarmmanager/alarmmanager.cpp
@@ -234,7 +234,6 @@ void processAlarm(const Alarm& calAlarm)
            			rewriteActiveLog (alarmList);
 				} catch (runtime_error& e)
 				{
-					if (ALARM_DEBUG) {                                
 						LoggingID lid(11);                                
 						MessageLog ml(lid);                                
 						Message msg;                                
@@ -242,9 +241,7 @@ void processAlarm(const Alarm& calAlarm)
 						args.add("rewriteActiveLog error:");                                
 						args.add(e.what());                                
 						msg.format(args);                                
-						ml.logDebugMessage(msg);                                
-					}
-					exit(1);
+						ml.logErrorMessage(msg);                                
 				}
            		break;
            	}
@@ -256,7 +253,6 @@ void processAlarm(const Alarm& calAlarm)
    			logAlarm (calAlarm, ACTIVE_ALARM_FILE);
 		} catch (runtime_error& e)
 		{
-			if (ALARM_DEBUG) {                                
 				LoggingID lid(11);                                
 				MessageLog ml(lid);                                
 				Message msg;                                
@@ -264,9 +260,7 @@ void processAlarm(const Alarm& calAlarm)
 				args.add("logAlarm error:");                                
 				args.add(e.what());                                
 				msg.format(args);                                
-				ml.logDebugMessage(msg);                                
-			}
-			exit(1);
+				ml.logErrorMessage(msg);                                
 		}
 	}
 	
@@ -276,7 +270,6 @@ void processAlarm(const Alarm& calAlarm)
 			logAlarm (calAlarm, ALARM_FILE);
 		} catch (runtime_error& e)
 		{
-			if (ALARM_DEBUG) {                                
 				LoggingID lid(11);                                
 				MessageLog ml(lid);                                
 				Message msg;                                
@@ -284,9 +277,7 @@ void processAlarm(const Alarm& calAlarm)
 				args.add("logAlarm error:");                                
 				args.add(e.what());                                
 				msg.format(args);                                
-				ml.logDebugMessage(msg);                                
-			}
-			exit(1);
+				ml.logErrorMessage(msg);                                
 		}
 	}
 }
@@ -462,7 +453,6 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st
   		configAlarm (calAlarm);
 	} catch (runtime_error& e)
 	{
-		if (ALARM_DEBUG) {                                
 			LoggingID lid(11);                                
 			MessageLog ml(lid);                                
 			Message msg;                                
@@ -470,9 +460,7 @@ void ALARMManager::sendAlarmReport (const char* componentID, int alarmID, int st
 			args.add("configAlarm error:");                                
 			args.add(e.what());                                
 			msg.format(args);                                
-			ml.logDebugMessage(msg);                                
-		}
-		exit(1);
+			ml.logErrorMessage(msg);                                
 	}
 
 	return;
diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp
index c8353d478..de2359000 100644
--- a/oamapps/postConfigure/postConfigure.cpp
+++ b/oamapps/postConfigure/postConfigure.cpp
@@ -814,10 +814,7 @@ int main(int argc, char *argv[])
 
 	//amazon install setup check
 	bool amazonInstall = false;
-	if (rootUser)
-		system("ec2-version > /tmp/amazon.log 2>&1");
-	else
-		system("sudo ec2-version > /tmp/amazon.log 2>&1");
+	system("ec2-version > /tmp/amazon.log 2>&1");
 
 	ifstream in("/tmp/amazon.log");
 
@@ -3401,7 +3398,7 @@ int main(int argc, char *argv[])
 					//run remote command script
 					cout << endl << "----- Starting MariaDB Columnstore on '" + remoteModuleName + "' -----" << endl << endl;
 
-					cmd = installDir + "/bin/remote_scp_put.sh " + remoteModuleIP + " " + installDir + "/etc/Columnstore.xml " +  installDir + "/etc/. > /dev/null 2>&1";
+					cmd = installDir + "/bin/remote_scp_put.sh " + remoteModuleIP + " " + installDir + "/etc/Columnstore.xml  > /dev/null 2>&1";
 					int rtnCode = system(cmd.c_str());
 
 					cmd = installDir + "/bin/remote_command.sh " + remoteModuleIP + " " + password +
@@ -5044,10 +5041,10 @@ bool updateBash()
 
 	if (!rootUser)
 	{
-		string cmd = "echo export columnstore_INSTALL_DIR=" + installDir + " >> " + fileName;
+		string cmd = "echo export COLUMNSTORE_INSTALL_DIR=" + installDir + " >> " + fileName;
 		system(cmd.c_str());
 	
-		cmd = "echo export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$columnstore_INSTALL_DIR/lib:$columnstore_INSTALL_DIR/mysql/lib/mysql >> " + fileName;
+		cmd = "echo export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql >> " + fileName;
 		system(cmd.c_str());
 	}
 

From 75fc2d9ae020353d84555ec4af6592d27854a6d2 Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Thu, 17 Nov 2016 14:42:53 +0000
Subject: [PATCH 23/54] mcol-404 - change the way that the log dir is setup

---
 oam/install_scripts/post-install | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install
index 6e245b31b..620c7ec87 100755
--- a/oam/install_scripts/post-install
+++ b/oam/install_scripts/post-install
@@ -95,15 +95,9 @@ sudo ln -s /usr/lib64/perl5/CORE/libperl.so /usr/lib64/libperl.so >/dev/null 2>&
 
 cd /
 
-if [ $user != "root" ]; then
-	sudo chmod 777 /var
-	sudo chmod 777 /var/log
-fi
-
-test -d /var/log/mariadb || mkdir /var/log/mariadb >/dev/null 2>&1
-sudo chmod 777 /var/log/mariadb
-test -d /var/log/mariadb/columnstore || mkdir /var/log/mariadb/columnstore >/dev/null 2>&1
-sudo chmod 777 /var/log/mariadb/columnstore
+test -d /var/log/mariadb || sudo mkdir /var/log/mariadb >/dev/null 2>&1
+test -d /var/log/mariadb/columnstore || sudo mkdir /var/log/mariadb/columnstore >/dev/null 2>&1
+sudo chmod -R 777 /var/log/mariadb
 
 test -d /var/log/mariadb/columnstore/archive || mkdir /var/log/mariadb/columnstore/archive >/dev/null 2>&1
 test -d /var/log/mariadb/columnstore/corefiles || mkdir /var/log/mariadb/columnstore/corefiles >/dev/null 2>&1

From 64a7695a3bd466adc702c51a3a14f1fc83d00250 Mon Sep 17 00:00:00 2001
From: Ben Thompson <dl3hp33ui@gmail.com>
Date: Thu, 17 Nov 2016 13:56:14 -0600
Subject: [PATCH 24/54] remove generated file releasenum and add it to
 gitignore file

---
 .gitignore       | 1 +
 build/releasenum | 2 --
 2 files changed, 1 insertion(+), 2 deletions(-)
 delete mode 100644 build/releasenum

diff --git a/.gitignore b/.gitignore
index c9dc8e53a..ec3d89ef1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -98,6 +98,7 @@ utils/loggingcpp/messageids.h
 CPackConfig.cmake
 CPackSourceConfig.cmake
 build/columnstore.community.spec
+build/releasenum
 *.rpm
 install_manifest_libs.txt
 install_manifest_platform.txt
diff --git a/build/releasenum b/build/releasenum
deleted file mode 100644
index 0aa2a30a8..000000000
--- a/build/releasenum
+++ /dev/null
@@ -1,2 +0,0 @@
-version=1.0.4
-release=1

From c1902ba577384d1fb32bcb10455834bb6d63cb61 Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Thu, 17 Nov 2016 20:04:20 +0000
Subject: [PATCH 25/54] mcol-61

---
 oamapps/postConfigure/postConfigure.cpp | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp
index de2359000..cbeb1da05 100644
--- a/oamapps/postConfigure/postConfigure.cpp
+++ b/oamapps/postConfigure/postConfigure.cpp
@@ -844,9 +844,6 @@ int main(int argc, char *argv[])
 			cloud  = oam::UnassignedName;
 		}
 
-		if ( cloud  == oam::UnassignedName )
-		    option = "2";
-		  
 		cout << "===== Amazon EC2-API-TOOLS Instance Install =====" << endl << endl;
 		cout << "You have 2 install options: " << endl << endl;
 		cout << "1. Utilizing the Amazon IDs for instances and volumes which allows for features like" << endl;
@@ -2933,8 +2930,9 @@ int main(int argc, char *argv[])
 					break;
 				}
 
-				if ( strcmp(pass1, "exit") == 0 )
+				if ( pass1 == "exit")
 					exit(0);
+
 				string p1 = pass1;
 				pass2=getpass("Confirm password > ");
 				string p2 = pass2;

From 8789ae7733f1dfa5f822ff47d8ecadb9074b246e Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Thu, 17 Nov 2016 20:06:03 +0000
Subject: [PATCH 26/54] mcol-61

---
 oam/install_scripts/post-install | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install
index 1bcdc9551..7fcccc17b 100755
--- a/oam/install_scripts/post-install
+++ b/oam/install_scripts/post-install
@@ -185,13 +185,11 @@ if [ $user = "root" ]; then
 	chmod 777 -R /dev/shm
 	rm -f /etc/default/columnstore
 else
-        sudo chmod 755 /var/log
-        sudo chmod -R 755 /var/log/mariadb/columnstore
+	sudo chmod 777 /tmp
 	sudo $installdir/bin/syslogSetup.sh --installdir=$installdir install > /tmp/syslog_install.log 2>&1
 	sudo chown $user:$user $installdir/etc/Columnstore.xml
 	sudo chmod -R 777 /dev/shm
 	sudo chmod 777 /var/lock/subsys
-	sudo chmod 777 /tmp
 	sudo rm -f /var/lock/subsys/mysql-Columnstore
 	sudo rm -f /tmp/StopColumnstore
 

From 9467a0887981e3515efa1fdd5a3b64a874532976 Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Thu, 17 Nov 2016 21:20:58 +0000
Subject: [PATCH 27/54] mcol-61 fix volumnes script with cplogger

---
 oam/cloud/MCSVolumeCmds.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/oam/cloud/MCSVolumeCmds.sh b/oam/cloud/MCSVolumeCmds.sh
index 1ee4dd024..7bac1a7dc 100644
--- a/oam/cloud/MCSVolumeCmds.sh
+++ b/oam/cloud/MCSVolumeCmds.sh
@@ -93,7 +93,7 @@ if [ "$1" = "createTag" ]; then
 fi
 
 
-test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions
+test -f $prefix/mariadb/columnstore/post/functions && . $prefix/mariadb/columnstore/post/functions
 
 ec2=`$prefix/mariadb/columnstore/bin/getConfig Installation EC2_HOME`
 
@@ -247,7 +247,7 @@ detachvolume() {
 			sleep 1
 		done
 		test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions
-		cplogger -w 100 "detachvolume failed: $STATUS"
+		$prefix/mariadb/columnstore/bin/cplogger -w 100 "detachvolume failed: $STATUS"
 		echo "failed"
 		exit 1
 	fi
@@ -258,7 +258,7 @@ detachvolume() {
 	fi
 
 	test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions
-	cplogger -w 100 "detachvolume failed status: $STATUS"
+	$prefix/mariadb/columnstore/bin/cplogger -w 100 "detachvolume failed status: $STATUS"
 	echo $STATUS
 	exit 1
 }
@@ -282,7 +282,7 @@ attachvolume() {
 			sleep 1
 		done
 		test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions
-		cplogger -w 100 "attachvolume failed: $STATUS"
+		$prefix/mariadb/columnstore/bin/cplogger -w 100 "attachvolume failed: $STATUS"
 		echo "failed"
 		exit 1
 	fi
@@ -293,7 +293,7 @@ attachvolume() {
 	fi
 
 	test -f /usr/local/mariadb/columnstore/post/functions && . /usr/local/mariadb/columnstore/post/functions
-	cplogger -w 100 "attachvolume failed: $STATUS"
+	$prefix/mariadb/columnstore/bin/cplogger -w 100 "attachvolume failed: $STATUS"
 	echo $STATUS
 	exit 1
 }

From 03efedde03814bda3074acff47e549cc2c2270af Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Thu, 17 Nov 2016 22:26:58 +0000
Subject: [PATCH 28/54] fix fstab non-root issue

---
 oam/oamcpp/liboamcpp.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp
index 2ca9e67d9..accd2d6ea 100644
--- a/oam/oamcpp/liboamcpp.cpp
+++ b/oam/oamcpp/liboamcpp.cpp
@@ -8549,8 +8549,11 @@ namespace oam
 		//update /etc/fstab with mount
 		string entry = device + " " + InstallDir + "/data" + dbrootID + " ext2 noatime,nodiratime,noauto 0 0";
 
-		//update local fstab	
-		cmd = "echo " + entry + " >> /etc/fstab";
+		//update local fstab
+        if (user == 0)
+			cmd = "echo " + entry + " >> /etc/fstab";
+        else
+			cmd = "sudo echo " + entry + " >> /etc/fstab";
 		system(cmd.c_str());
 
 		//use from addmodule later

From f785bea1d9f9ae63d06b7effeb23010e17619789 Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Thu, 17 Nov 2016 22:42:22 +0000
Subject: [PATCH 29/54] mcol-61

---
 oam/oamcpp/liboamcpp.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp
index accd2d6ea..80021cddb 100644
--- a/oam/oamcpp/liboamcpp.cpp
+++ b/oam/oamcpp/liboamcpp.cpp
@@ -8550,6 +8550,9 @@ namespace oam
 		string entry = device + " " + InstallDir + "/data" + dbrootID + " ext2 noatime,nodiratime,noauto 0 0";
 
 		//update local fstab
+		int user;
+        user = getuid();
+
         if (user == 0)
 			cmd = "echo " + entry + " >> /etc/fstab";
         else

From 213eb28f5b80b6b32074edf69d84e0e5fb91c5be Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Fri, 18 Nov 2016 17:40:44 +0000
Subject: [PATCH 30/54] mcol-61

---
 oam/install_scripts/binary_installer.sh |  2 +-
 oam/install_scripts/module_installer.sh | 25 +++++++++++++------------
 oam/install_scripts/post-install        |  1 +
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/oam/install_scripts/binary_installer.sh b/oam/install_scripts/binary_installer.sh
index ea6123f13..efd202cf1 100644
--- a/oam/install_scripts/binary_installer.sh
+++ b/oam/install_scripts/binary_installer.sh
@@ -234,7 +234,7 @@ if { $INSTALLTYPE == "initial"} {
 	send_user "Run Module Installer                            "
 	send " \n"
 	send date\n
-	send "ssh $USERNAME@$SERVER '$INSTALLDIR/bin/module_installer.sh --module=$MODULETYPE --port=$MYSQLPORT --installdir=$INSTALLDIR $MODULETYPE'\n"
+	send "ssh $USERNAME@$SERVER '$INSTALLDIR/bin/module_installer.sh --module=$MODULETYPE --port=$MYSQLPORT --installdir=$INSTALLDIR'\n"
 	set timeout 10
 	expect {
 		"word: " { send "$PASSWORD\n" }
diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh
index 5b0fb2e01..6765fc14c 100755
--- a/oam/install_scripts/module_installer.sh
+++ b/oam/install_scripts/module_installer.sh
@@ -53,6 +53,19 @@ fi
 
 export COLUMNSTORE_INSTALL_DIR=$installdir
 
+if [ $user != "root" ]; then
+        echo "Setup .bashrc on Module for non-root"
+
+        eval userhome=~$user
+        bashFile=$userhome/.bashrc
+        touch ${bashFile}
+
+        echo " " >> ${bashFile}
+        echo "export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR" >> ${bashFile}
+        echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql" >> ${bashFile}
+	. ${bashFile}
+fi
+
 cloud=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation Cloud`
 if [ $module = "pm" ]; then
 	if [ $cloud = "amazon-ec2" ] || [ $cloud = "amazon-vpc" ]; then
@@ -113,18 +126,6 @@ if [ $EUID -eq 0 -a -f $COLUMNSTORE_INSTALL_DIR/local/rc.local.columnstore ]; th
 	fi
 fi
 
-if [ $user != "root" ]; then
-	echo "Setup .bashrc on Module for non-root"
-
-	eval userhome=~$user
-	bashFile=$userhome/.bashrc
-	touch ${bashFile}
-
-	echo " " >> ${bashFile}
-	echo "export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR" >> ${bashFile}
-	echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql" >> ${bashFile}
-fi
-
 plugin=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig SystemConfig DataFilePlugin`
 if [ -n "$plugin" ]; then
 	echo "Setup .bashrc on Module for local-query"
diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install
index 7fcccc17b..aff6b8f1a 100755
--- a/oam/install_scripts/post-install
+++ b/oam/install_scripts/post-install
@@ -186,6 +186,7 @@ if [ $user = "root" ]; then
 	rm -f /etc/default/columnstore
 else
 	sudo chmod 777 /tmp
+	sudo rm -f /tmp/syslog_install.log
 	sudo $installdir/bin/syslogSetup.sh --installdir=$installdir install > /tmp/syslog_install.log 2>&1
 	sudo chown $user:$user $installdir/etc/Columnstore.xml
 	sudo chmod -R 777 /dev/shm

From 2638c05e26341862b06fbf273cae79a754b6f8f8 Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Fri, 18 Nov 2016 17:55:18 +0000
Subject: [PATCH 31/54] mcol-61

---
 oam/install_scripts/rsync.sh      | 9 +++++++--
 oamapps/postConfigure/helpers.cpp | 8 ++++----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/oam/install_scripts/rsync.sh b/oam/install_scripts/rsync.sh
index 2fa53ee6b..476393eb4 100755
--- a/oam/install_scripts/rsync.sh
+++ b/oam/install_scripts/rsync.sh
@@ -6,7 +6,6 @@
 # Argument 1 - Remote Server Host Name or IP address
 # Argument 2 - Root Password of remote server
 # Argument 3 - Debug flag 1 for on, 0 for off
-set USERNAME "root@"
 set SERVER [lindex $argv 0]
 set PASSWORD [lindex $argv 1]
 set INSTALLDIR [lindex $argv 2]
@@ -15,13 +14,19 @@ set DEBUG [lindex $argv 3]
 log_user $DEBUG
 spawn -noecho /bin/bash
 
+if {[info exists env(USER)]} {
+    set USERNAME $env(USER)
+} else {
+    set USERNAME "root"
+}
+
 
 if { $PASSWORD == "ssh" } {
 	set USERNAME ""
 	set PASSWORD ""
 }
 
-set COMMAND "rsync -vopgr -e ssh --exclude=mysql/ --exclude=test/ --exclude=infinidb_vtable/ --exclude=infinidb_querystats/ --exclude=calpontsys/ --include=*/ --include=*/* --exclude=* $INSTALLDIR/mysql/db/ $USERNAME$SERVER:$INSTALLDIR/mysql/db/"
+set COMMAND "rsync -vopgr -e ssh --exclude=mysql/ --exclude=test/ --exclude=infinidb_vtable/ --exclude=infinidb_querystats/ --exclude=calpontsys/ --include=*/ --include=*/* --exclude=* $INSTALLDIR/mysql/db/ $USERNAME$@SERVER:$INSTALLDIR/mysql/db/"
 
 #
 # run command
diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp
index 36cca5177..c2157bc5d 100644
--- a/oamapps/postConfigure/helpers.cpp
+++ b/oamapps/postConfigure/helpers.cpp
@@ -491,7 +491,7 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st
 							returnStatus = sendMsgProcMon( (*pt).DeviceName, msg1, requestID, 600 );
 			
 							if ( returnStatus != API_SUCCESS) {
-								cout << "ERROR: Error return in running the MariaDB Columnstore Master DB Distribute, check /tmp/master-dist.logs on " << masterModule << endl;
+								cout << endl << "ERROR: Error return in running the MariaDB Columnstore Master DB Distribute, check /tmp/master-dist.logs on " << masterModule << endl;
 								return returnStatus;
 							}
 
@@ -503,7 +503,7 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st
 							returnStatus = sendMsgProcMon( (*pt).DeviceName, msg, requestID, 30 );
 			
 							if ( returnStatus != API_SUCCESS) {
-								cout << "ERROR: Error return in running the MariaDB Columnstore Master replication, check /tmp/master-rep*.logs on " << masterModule << endl;
+								cout << endl << "ERROR: Error return in running the MariaDB Columnstore Master replication, check /tmp/master-rep*.logs on " << masterModule << endl;
 								return returnStatus;
 							}
 
@@ -534,7 +534,7 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st
 							returnStatus = sendMsgProcMon( (*pt).DeviceName, msg, requestID, 30 );
 			
 							if ( returnStatus != API_SUCCESS) {
-								cout << "ERROR: Error return in running the MariaDB Columnstore Slave replication, check /tmp/slave-rep*.logs on " << (*pt).DeviceName << endl;
+								cout << endl << "ERROR: Error return in running the MariaDB Columnstore Slave replication, check /tmp/slave-rep*.logs on " << (*pt).DeviceName << endl;
 								return returnStatus;
 							}
 
@@ -543,7 +543,7 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st
 					}
 					else
 					{
-						cout << "ERROR: Module not Active, replication not done on " << (*pt).DeviceName << endl;
+						cout << endl << "ERROR: Module not Active, replication not done on " << (*pt).DeviceName << endl;
 						pt++;
 					}
 				}

From 23d38aa50398798801ec688af70a9eb010b995c4 Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Fri, 18 Nov 2016 19:55:48 +0000
Subject: [PATCH 32/54] mcol-61 and mcol-404 fix

---
 oam/install_scripts/post-install        | 10 +++++-----
 oam/install_scripts/rsync.sh            |  2 +-
 oamapps/postConfigure/helpers.cpp       |  2 +-
 oamapps/postConfigure/postConfigure.cpp |  6 +-----
 4 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install
index aff6b8f1a..145c5c010 100755
--- a/oam/install_scripts/post-install
+++ b/oam/install_scripts/post-install
@@ -250,7 +250,7 @@ The next steps are:
 
 export COLUMNSTORE_INSTALL_DIR=$installdir
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib/mysql
-$installdir/bin/postConfigure -i $installdir
+$installdir/bin/postConfigure --installdir=$installdir
 
 EOD
 	else
@@ -282,13 +282,13 @@ If you are intending to install MariaDB Columnstore over Hadoop, the next steps
 export COLUMNSTORE_INSTALL_DIR=$installdir
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib/mysql
 . $installdir/bin/setenv-hdfs-20
-$installdir/bin/postConfigure -i $installdir
+$installdir/bin/postConfigure --installdir=$installdir
 
 If you are intending to install MariaDB Columnstore without Hadoop, the next steps are:
 
 export COLUMNSTORE_INSTALL_DIR=$installdir
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib/mysql
-$installdir/bin/postConfigure -i $installdir
+$installdir/bin/postConfigure --installdir=$installdir
 
 EOD
 		else
@@ -321,13 +321,13 @@ If you are intending to install MariaDB Columnstore over Hadoop, the next steps
 export COLUMNSTORE_INSTALL_DIR=$installdir
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib/mysql:$libpath
 . $installdir/bin/setenv-hdfs-12
-$installdir/bin/postConfigure -i $installdir
+$installdir/bin/postConfigure --installdir=$installdir
 
 If you are intending to install MariaDB Columnstore without Hadoop, the next steps are:
 
 export COLUMNSTORE_INSTALL_DIR=$installdir
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib/mysql
-$installdir/bin/postConfigure -i $installdir
+$installdir/bin/postConfigure --installdir=$installdir
 
 EOD
 			else
diff --git a/oam/install_scripts/rsync.sh b/oam/install_scripts/rsync.sh
index 476393eb4..d7d73d2d0 100755
--- a/oam/install_scripts/rsync.sh
+++ b/oam/install_scripts/rsync.sh
@@ -26,7 +26,7 @@ if { $PASSWORD == "ssh" } {
 	set PASSWORD ""
 }
 
-set COMMAND "rsync -vopgr -e ssh --exclude=mysql/ --exclude=test/ --exclude=infinidb_vtable/ --exclude=infinidb_querystats/ --exclude=calpontsys/ --include=*/ --include=*/* --exclude=* $INSTALLDIR/mysql/db/ $USERNAME$@SERVER:$INSTALLDIR/mysql/db/"
+set COMMAND "rsync -vopgr -e ssh --exclude=mysql/ --exclude=test/ --exclude=infinidb_vtable/ --exclude=infinidb_querystats/ --exclude=calpontsys/ --include=*/ --include=*/* --exclude=* $INSTALLDIR/mysql/db/ $USERNAME@$SERVER:$INSTALLDIR/mysql/db/"
 
 #
 # run command
diff --git a/oamapps/postConfigure/helpers.cpp b/oamapps/postConfigure/helpers.cpp
index c2157bc5d..a0a2d5825 100644
--- a/oamapps/postConfigure/helpers.cpp
+++ b/oamapps/postConfigure/helpers.cpp
@@ -491,7 +491,7 @@ int sendReplicationRequest(int IserverTypeInstall, std::string password, std::st
 							returnStatus = sendMsgProcMon( (*pt).DeviceName, msg1, requestID, 600 );
 			
 							if ( returnStatus != API_SUCCESS) {
-								cout << endl << "ERROR: Error return in running the MariaDB Columnstore Master DB Distribute, check /tmp/master-dist.logs on " << masterModule << endl;
+								cout << endl << "ERROR: Error return in running the MariaDB Columnstore Master DB Distribute, check /tmp/master-dist*.logs on " << masterModule << endl;
 								return returnStatus;
 							}
 
diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp
index cbeb1da05..50bed8e64 100644
--- a/oamapps/postConfigure/postConfigure.cpp
+++ b/oamapps/postConfigure/postConfigure.cpp
@@ -336,17 +336,13 @@ int main(int argc, char *argv[])
 		// for backward compatibility
 		else if( string("-n") == argv[i] )
 			noPrompting = true;
-		else if( string("-i") == argv[i] ) {
+		else if( string("--installdir=") == argv[i] ) {
 			i++;
 			if (i >= argc ) {
 				cout << "   ERROR: install dir not provided" << endl;
 				exit (1);
 			}
 			installDir = argv[i];
-			if ( installDir.find("-") != string::npos ) {
-				cout << "   ERROR: Valid install dir not provided" << endl;
-				exit (1);
-			}			
 		}
 		else if( string("-port") == argv[i] ) {
 			i++;

From f4766b2625eaebab722b0d36448b5a9529e9690f Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Fri, 18 Nov 2016 20:21:16 +0000
Subject: [PATCH 33/54] more changes for mcol-61

---
 oam/install_scripts/module_installer.sh | 13 -------------
 oam/install_scripts/post-install        | 22 +++++++++++-----------
 oamapps/postConfigure/postConfigure.cpp |  9 ---------
 3 files changed, 11 insertions(+), 33 deletions(-)

diff --git a/oam/install_scripts/module_installer.sh b/oam/install_scripts/module_installer.sh
index 6765fc14c..19d76bc6f 100755
--- a/oam/install_scripts/module_installer.sh
+++ b/oam/install_scripts/module_installer.sh
@@ -53,19 +53,6 @@ fi
 
 export COLUMNSTORE_INSTALL_DIR=$installdir
 
-if [ $user != "root" ]; then
-        echo "Setup .bashrc on Module for non-root"
-
-        eval userhome=~$user
-        bashFile=$userhome/.bashrc
-        touch ${bashFile}
-
-        echo " " >> ${bashFile}
-        echo "export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR" >> ${bashFile}
-        echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql" >> ${bashFile}
-	. ${bashFile}
-fi
-
 cloud=`$COLUMNSTORE_INSTALL_DIR/bin/getConfig Installation Cloud`
 if [ $module = "pm" ]; then
 	if [ $cloud = "amazon-ec2" ] || [ $cloud = "amazon-vpc" ]; then
diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install
index 145c5c010..2473f94f7 100755
--- a/oam/install_scripts/post-install
+++ b/oam/install_scripts/post-install
@@ -84,17 +84,17 @@ fi
 # softlink for libperl.sp, used by cplogger
 sudo ln -s /usr/lib64/perl5/CORE/libperl.so /usr/lib64/libperl.so >/dev/null 2>&1
 
-# TODO-based on early experience with HDP it doesn't install bigtop
-# so the code below will not correctly detect JAVA_HOME.  Need to 
-# decide how to handle
-# Autodetect JAVA_HOME if not defined
-#if [ -e /usr/libexec/bigtop-detect-javahome ]; then
-#. /usr/libexec/bigtop-detect-javahome
-#elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
-#. /usr/lib/bigtop-utils/bigtop-detect-javahome
-#fi
-#ln -sf $JAVA_HOME/jre/lib/amd64/server/libjvm.so
-#chown -h $user.$user libjvm.so >/dev/null 2>&1
+if [ $user != "root" ]; then
+        eval userhome=~$user
+        bashFile=$userhome/.bashrc
+        touch ${bashFile}
+	/bin/cp -f ${bashFile} ${bashFile}.columnstoreSave > /dev/null 2>&1
+
+        echo " " >> ${bashFile}
+        echo "export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR" >> ${bashFile}
+        echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql" >> ${bashFile}
+        . ${bashFile}
+fi
 
 cd /
 
diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp
index 50bed8e64..c3d68955a 100644
--- a/oamapps/postConfigure/postConfigure.cpp
+++ b/oamapps/postConfigure/postConfigure.cpp
@@ -5033,15 +5033,6 @@ bool updateBash()
 
    	ifstream newFile (fileName.c_str());
 
-	if (!rootUser)
-	{
-		string cmd = "echo export COLUMNSTORE_INSTALL_DIR=" + installDir + " >> " + fileName;
-		system(cmd.c_str());
-	
-		cmd = "echo export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql >> " + fileName;
-		system(cmd.c_str());
-	}
-
 	if ( hdfs ) 
 	{	
 		string cmd = "echo . " + installDir + "/bin/" + DataFileEnvFile + " >> " + fileName;

From d2ee5b06aecee00e093b99ee21dadff2e09407aa Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Fri, 18 Nov 2016 21:00:27 +0000
Subject: [PATCH 34/54] more non-root changes, mcol-404

---
 oam/install_scripts/post-install  | 34 +++++++++++++++++++++++--------
 oam/install_scripts/pre-uninstall | 24 ++++++++++++----------
 2 files changed, 39 insertions(+), 19 deletions(-)

diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install
index 2473f94f7..a1e071b93 100755
--- a/oam/install_scripts/post-install
+++ b/oam/install_scripts/post-install
@@ -84,16 +84,34 @@ fi
 # softlink for libperl.sp, used by cplogger
 sudo ln -s /usr/lib64/perl5/CORE/libperl.so /usr/lib64/libperl.so >/dev/null 2>&1
 
-if [ $user != "root" ]; then
-        eval userhome=~$user
-        bashFile=$userhome/.bashrc
-        touch ${bashFile}
-	/bin/cp -f ${bashFile} ${bashFile}.columnstoreSave > /dev/null 2>&1
+#setup .bashrc
+eval userhome=~$user
+bashFile=$userhome/.bashrc
+touch ${bashFile}
+/bin/cp -f ${bashFile} ${bashFile}.columnstoreSave > /dev/null 2>&1
 
+if [ $user != "root" ]; then
+
+	egrep -qs 'MariaDB Columnstore Non-Root' ${bashFile}
+	rc1=$?
+	if [ $rc1 -ne 0 ]; then
+        	echo " " >> ${bashFile}
+		echo "MariaDB Columnstore Non-Root Environment Variables" >> ${bashFile}
+        	echo "export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR" >> ${bashFile}
+        	echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql" >> ${bashFile}
+        	. ${bashFile}
+fi
+
+if [ $installdir != "/usr/local/mariadb/columnstore" ]; then
+        sed -i -e s@/usr/local/mariadb/columnstore@$installdir@g $installdir/bin/columnstoreAlias
+fi
+egrep -qs 'MariaDB Columnstore Database Platform Alias Commands' ${bashFile}
+rc1=$?
+egrep -qs 'MariaDB Columnstore Alias Commands' ${bashFile}
+rc2=$?
+if [ $rc1 -ne 0 -a $rc2 -ne 0 ]; then
+        cat $installdir/bin/columnstoreAlias >> ${bashFile}
         echo " " >> ${bashFile}
-        echo "export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR" >> ${bashFile}
-        echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql" >> ${bashFile}
-        . ${bashFile}
 fi
 
 cd /
diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall
index bf3bbc053..d81e9c513 100755
--- a/oam/install_scripts/pre-uninstall
+++ b/oam/install_scripts/pre-uninstall
@@ -33,22 +33,13 @@ for arg in "$@"; do
 	fi
 done
 
+. /home/$user/.bashrc
+
 #stop services
 $installdir/bin/columnstore stop > /dev/null 2>&1
 $installdir/myql/columnstore-Mysql stop > /dev/null 2>&1
 
 # uninstall OS scripts updated by postConfigure
-if [ $user = "root" ]; then
-	if test -f /root/.bashrc.columnstoreSave ; then
-		mv -f /root/.bashrc.columnstoreSave /root/.bashrc > /dev/null 2>&1
-	fi
-else
-        if test -f $HOME/.bashrc.columnstoreSave ; then
-                mv -f /home/$user/.bashrc.columnstoreSave /home/$user/.bashrc > /dev/null 2>&1
-        fi
-fi
-	
-
 if test -f /etc/exports.columnstoreSave ; then
 	$sudo mv -f /etc/exports.columnstoreSave /etc/exports > /dev/null 2>&1
 else
@@ -173,6 +164,17 @@ if [ $installdir == "/usr/local/mariadb/columnstore" ]; then
         sudo ldconfig
 fi
 
+#cleanup .bashrc	
+if [ $user = "root" ]; then
+        if test -f /root/.bashrc.columnstoreSave ; then
+                mv -f /root/.bashrc.columnstoreSave /root/.bashrc > /dev/null 2>&1
+        fi
+else
+        if test -f $HOME/.bashrc.columnstoreSave ; then
+                mv -f /home/$user/.bashrc.columnstoreSave /home/$user/.bashrc > /dev/null 2>&1
+        fi
+fi
+
 #tell user to run post configure script
 echo " "
 echo "Mariab Columnstore uninstall completed"

From b4e617c2c04974839553af404636390f920d6140 Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Fri, 18 Nov 2016 21:04:46 +0000
Subject: [PATCH 35/54] more non-root changes, mcol-404

---
 oam/install_scripts/post-install | 1 +
 1 file changed, 1 insertion(+)

diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install
index a1e071b93..d526a7d14 100755
--- a/oam/install_scripts/post-install
+++ b/oam/install_scripts/post-install
@@ -100,6 +100,7 @@ if [ $user != "root" ]; then
         	echo "export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR" >> ${bashFile}
         	echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql" >> ${bashFile}
         	. ${bashFile}
+	fi
 fi
 
 if [ $installdir != "/usr/local/mariadb/columnstore" ]; then

From 4ceb1d93355f8d4700975d25da704518d10e6b05 Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Fri, 18 Nov 2016 22:18:08 +0000
Subject: [PATCH 36/54] change back the -i option

---
 oam/install_scripts/post-install        | 12 ++++++------
 oamapps/postConfigure/postConfigure.cpp | 16 ++++++++--------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install
index d526a7d14..400c6844b 100755
--- a/oam/install_scripts/post-install
+++ b/oam/install_scripts/post-install
@@ -96,7 +96,7 @@ if [ $user != "root" ]; then
 	rc1=$?
 	if [ $rc1 -ne 0 ]; then
         	echo " " >> ${bashFile}
-		echo "MariaDB Columnstore Non-Root Environment Variables" >> ${bashFile}
+		echo "# MariaDB Columnstore Non-Root Environment Variables" >> ${bashFile}
         	echo "export COLUMNSTORE_INSTALL_DIR=$COLUMNSTORE_INSTALL_DIR" >> ${bashFile}
         	echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$COLUMNSTORE_INSTALL_DIR/lib:$COLUMNSTORE_INSTALL_DIR/mysql/lib/mysql" >> ${bashFile}
         	. ${bashFile}
@@ -269,7 +269,7 @@ The next steps are:
 
 export COLUMNSTORE_INSTALL_DIR=$installdir
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib/mysql
-$installdir/bin/postConfigure --installdir=$installdir
+$installdir/bin/postConfigure -i $installdir
 
 EOD
 	else
@@ -301,13 +301,13 @@ If you are intending to install MariaDB Columnstore over Hadoop, the next steps
 export COLUMNSTORE_INSTALL_DIR=$installdir
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib/mysql
 . $installdir/bin/setenv-hdfs-20
-$installdir/bin/postConfigure --installdir=$installdir
+$installdir/bin/postConfigure -i $installdir
 
 If you are intending to install MariaDB Columnstore without Hadoop, the next steps are:
 
 export COLUMNSTORE_INSTALL_DIR=$installdir
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib/mysql
-$installdir/bin/postConfigure --installdir=$installdir
+$installdir/bin/postConfigure -i $installdir
 
 EOD
 		else
@@ -340,13 +340,13 @@ If you are intending to install MariaDB Columnstore over Hadoop, the next steps
 export COLUMNSTORE_INSTALL_DIR=$installdir
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib/mysql:$libpath
 . $installdir/bin/setenv-hdfs-12
-$installdir/bin/postConfigure --installdir=$installdir
+$installdir/bin/postConfigure -i $installdir
 
 If you are intending to install MariaDB Columnstore without Hadoop, the next steps are:
 
 export COLUMNSTORE_INSTALL_DIR=$installdir
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$installdir/lib:$installdir/mysql/lib/mysql
-$installdir/bin/postConfigure --installdir=$installdir
+$installdir/bin/postConfigure -i $installdir
 
 EOD
 			else
diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp
index c3d68955a..0a103c591 100644
--- a/oamapps/postConfigure/postConfigure.cpp
+++ b/oamapps/postConfigure/postConfigure.cpp
@@ -336,14 +336,6 @@ int main(int argc, char *argv[])
 		// for backward compatibility
 		else if( string("-n") == argv[i] )
 			noPrompting = true;
-		else if( string("--installdir=") == argv[i] ) {
-			i++;
-			if (i >= argc ) {
-				cout << "   ERROR: install dir not provided" << endl;
-				exit (1);
-			}
-			installDir = argv[i];
-		}
 		else if( string("-port") == argv[i] ) {
 			i++;
 			if (i >= argc ) {
@@ -357,6 +349,14 @@ int main(int argc, char *argv[])
 				exit (1);
 			}
 		}
+        else if( string("-i") == argv[i] ) {
+            i++;
+            if (i >= argc ) {
+                cout << "   ERROR: Path not provided" << endl;
+                exit (1);
+            }
+            installDir = argv[i];
+        }
 		else
 		{
 			cout << "   ERROR: Invalid Argument = " << argv[i] << endl;

From 88ca71a15f621a5aeb9df55482e997fa8e3edecc Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Sat, 19 Nov 2016 22:15:03 +0000
Subject: [PATCH 37/54] fix get private ip

---
 oam/cloud/MCSInstanceCmds.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oam/cloud/MCSInstanceCmds.sh b/oam/cloud/MCSInstanceCmds.sh
index 3e648abc3..225fcbf69 100644
--- a/oam/cloud/MCSInstanceCmds.sh
+++ b/oam/cloud/MCSInstanceCmds.sh
@@ -259,7 +259,7 @@ getPrivateIP() {
 	if [ "$subnet" == "unassigned" ]; then
 		IpAddr=`head -n 2 /tmp/instanceInfo_$instanceName | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $14}'`
 	else
-		IpAddr=`head -n 2 /tmp/instanceInfo_$instanceName | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $13}'`
+		IpAddr=`head -n 2 /tmp/instanceInfo_$instanceName | awk '{gsub(/^[ \t]+|[ \t]+$/,"");print $14}'`
 	fi
 
 	echo $IpAddr

From 32db3bfc86729e44499cc96b0c813f022f47c2d0 Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Sun, 20 Nov 2016 00:07:09 +0000
Subject: [PATCH 38/54] mcol-61 - change to clean out /tmp

---
 oam/install_scripts/post-install | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install
index 400c6844b..f00360731 100755
--- a/oam/install_scripts/post-install
+++ b/oam/install_scripts/post-install
@@ -205,13 +205,13 @@ if [ $user = "root" ]; then
 	rm -f /etc/default/columnstore
 else
 	sudo chmod 777 /tmp
-	sudo rm -f /tmp/syslog_install.log
+	sudo rm -f /tmp/*
 	sudo $installdir/bin/syslogSetup.sh --installdir=$installdir install > /tmp/syslog_install.log 2>&1
 	sudo chown $user:$user $installdir/etc/Columnstore.xml
 	sudo chmod -R 777 /dev/shm
 	sudo chmod 777 /var/lock/subsys
 	sudo rm -f /var/lock/subsys/mysql-Columnstore
-	sudo rm -f /tmp/StopColumnstore
+	sudo chmod 777 /etc/fstab	
 
 	sed -i -e s@/usr/local/mariadb/columnstore@$installdir@g $installdir/bin/columnstore.def
 	sudo cp $installdir/bin/columnstore.def /etc/default/columnstore

From e29f7b22ea16e3b4a64567229bd21cbb56e9b826 Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Sun, 20 Nov 2016 00:10:21 +0000
Subject: [PATCH 39/54] fix non-root fstab permission issue

---
 oamapps/postConfigure/postConfigure.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp
index 0a103c591..8b049f363 100644
--- a/oamapps/postConfigure/postConfigure.cpp
+++ b/oamapps/postConfigure/postConfigure.cpp
@@ -4981,7 +4981,12 @@ void setSystemName()
  */
 bool copyFstab(string moduleName)
 {
-	string cmd = "/bin/cp -f /etc/fstab " + installDir + "/local/etc/" + moduleName + "/. > /dev/null 2>&1";
+	string cmd;	
+	if ( rootUser)
+   		cmd = "/bin/cp -f /etc/fstab " + installDir + "/local/etc/" + moduleName + "/. > /dev/null 2>&1";
+	else
+		cmd = "/sudo bin/cp -f /etc/fstab " + installDir + "/local/etc/" + moduleName + "/. > /dev/null 2>&1";
+
 	system(cmd.c_str());
 
 	return true;

From 2f366aa1bc6780af87a65e9bb140b9196651aa21 Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Sun, 20 Nov 2016 00:12:49 +0000
Subject: [PATCH 40/54] fix non-root fstab permission issue

---
 oam/oamcpp/liboamcpp.cpp | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp
index 80021cddb..736ec2e06 100644
--- a/oam/oamcpp/liboamcpp.cpp
+++ b/oam/oamcpp/liboamcpp.cpp
@@ -8540,8 +8540,13 @@ namespace oam
 	{
 		writeLog("updateFstab called: " + device + ":" + dbrootID, LOG_TYPE_DEBUG );
 
-		//check if entry already exist 
-		string cmd = "grep /data" + dbrootID + " /etc/fstab > /dev/null 2>&1";
+		//check if entry already exist
+		string cmd;
+		if (user == 0)
+			cmd = "grep /data" + dbrootID + " /etc/fstab > /dev/null 2>&1";
+		else
+            cmd = "sudo grep /data" + dbrootID + " /etc/fstab > /dev/null 2>&1";
+
 		int status = system(cmd.c_str());
 		if (WEXITSTATUS(status) == 0 )
 			return "";

From ac5940fed381c6dbe96e8ec48686b9ee3aa2eb5e Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Sun, 20 Nov 2016 00:14:38 +0000
Subject: [PATCH 41/54] fix non-root fstab permission issue

---
 oam/oamcpp/liboamcpp.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp
index 736ec2e06..14f2d8038 100644
--- a/oam/oamcpp/liboamcpp.cpp
+++ b/oam/oamcpp/liboamcpp.cpp
@@ -8541,6 +8541,9 @@ namespace oam
 		writeLog("updateFstab called: " + device + ":" + dbrootID, LOG_TYPE_DEBUG );
 
 		//check if entry already exist
+        int user;
+        user = getuid();
+
 		string cmd;
 		if (user == 0)
 			cmd = "grep /data" + dbrootID + " /etc/fstab > /dev/null 2>&1";
@@ -8555,9 +8558,6 @@ namespace oam
 		string entry = device + " " + InstallDir + "/data" + dbrootID + " ext2 noatime,nodiratime,noauto 0 0";
 
 		//update local fstab
-		int user;
-        user = getuid();
-
         if (user == 0)
 			cmd = "echo " + entry + " >> /etc/fstab";
         else

From 3affcd465d3a647b90d00b3450973b84cbc80fd1 Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Sun, 20 Nov 2016 01:23:59 +0000
Subject: [PATCH 42/54] clean out /tmp for non-root

---
 oam/install_scripts/post-install | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install
index f00360731..564fd87dc 100755
--- a/oam/install_scripts/post-install
+++ b/oam/install_scripts/post-install
@@ -205,7 +205,7 @@ if [ $user = "root" ]; then
 	rm -f /etc/default/columnstore
 else
 	sudo chmod 777 /tmp
-	sudo rm -f /tmp/*
+	sudo rm -fr /tmp/* > /dev/null 2>&1
 	sudo $installdir/bin/syslogSetup.sh --installdir=$installdir install > /tmp/syslog_install.log 2>&1
 	sudo chown $user:$user $installdir/etc/Columnstore.xml
 	sudo chmod -R 777 /dev/shm

From 02a5624cb34674d1fc1f526f574c191187438c36 Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Sun, 20 Nov 2016 16:29:42 +0000
Subject: [PATCH 43/54] MCOL-404 AND MCOL-61

---
 oam/install_scripts/master-rep-columnstore.sh |  7 +-
 oam/install_scripts/slave-rep-columnstore.sh  |  7 +-
 oamapps/postConfigure/postConfigure.cpp       | 72 ++++++-------------
 3 files changed, 32 insertions(+), 54 deletions(-)

diff --git a/oam/install_scripts/master-rep-columnstore.sh b/oam/install_scripts/master-rep-columnstore.sh
index a4b0992ed..16b956d29 100644
--- a/oam/install_scripts/master-rep-columnstore.sh
+++ b/oam/install_scripts/master-rep-columnstore.sh
@@ -35,10 +35,13 @@ for arg in "$@"; do
 	fi
 done
 
+if { $PASSWORD == "ssh" } {
+	set PASSWORD ""
+}
+
 test -f $installdir/post/functions && . $installdir/post/functions
 
-repUser="idbrep"
-password="Calpont1"
+repUser="mcsrep"
 
 >/tmp/master-rep-status-$hostipaddr.log
 
diff --git a/oam/install_scripts/slave-rep-columnstore.sh b/oam/install_scripts/slave-rep-columnstore.sh
index 000dc532a..6054e056b 100644
--- a/oam/install_scripts/slave-rep-columnstore.sh
+++ b/oam/install_scripts/slave-rep-columnstore.sh
@@ -41,10 +41,13 @@ for arg in "$@"; do
 	fi
 done
 
+if { $PASSWORD == "ssh" } {
+	set PASSWORD ""
+}
+
 test -f $installdir/post/functions && . $installdir/post/functions
 
-repUser="idbrep"
-password="Calpont1"
+repUser="mscrep"
 
 >/tmp/slave-rep-status.log
 
diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp
index 8b049f363..6862597bd 100644
--- a/oamapps/postConfigure/postConfigure.cpp
+++ b/oamapps/postConfigure/postConfigure.cpp
@@ -209,6 +209,7 @@ int main(int argc, char *argv[])
 	noPrompting = false;
 	string password;
 	string cmd;
+	bool disableAmazon = false;
 //  	struct sysinfo myinfo; 
 
 	// hidden options
@@ -276,15 +277,17 @@ int main(int argc, char *argv[])
 			cout << "	Enter one of the options within [], if available, or" << endl;
 			cout << "	Enter a new value" << endl << endl;
 			cout << endl;
-   			cout << "Usage: postConfigure [-h][-c][-u][-p][-mp][-s][-port]" << endl;
+   			cout << "Usage: postConfigure [-h][-c][-u][-p][-mp][-s][-port][-i][-da]" << endl;
 			cout << "   -h  Help" << endl;
 			cout << "   -c  Config File to use to extract configuration data, default is Columnstore.xml.rpmsave" << endl;
 			cout << "   -u  Upgrade, Install using the Config File from -c, default to Columnstore.xml.rpmsave" << endl;
-			cout << "	 If ssh-keys aren't setup, you should provide passwords as command line arguments" << endl;
+			cout << "	    If ssh-keys aren't setup, you should provide passwords as command line arguments" << endl;
 			cout << "   -p  Unix Password, used with no-prompting option" << endl;
 			cout << "   -mp MariaDB Columnstore Password" << endl;
 			cout << "   -s  Single Threaded Remote Install" << endl;
 			cout << "   -port MariaDB Columnstore Port Address" << endl;
+            cout << "   -i Non-root Install directory, Only use for non-root installs" << endl;
+            cout << "   -da Disable Amazon functionality, install using Stardard Hostnames and IP Addresses" << endl;
 			exit (0);
 		}
       		else if( string("-s") == argv[i] )
@@ -349,6 +352,8 @@ int main(int argc, char *argv[])
 				exit (1);
 			}
 		}
+        else if( string("da") == argv[i] )
+            disableAmazon = true;
         else if( string("-i") == argv[i] ) {
             i++;
             if (i >= argc ) {
@@ -810,27 +815,29 @@ int main(int argc, char *argv[])
 
 	//amazon install setup check
 	bool amazonInstall = false;
-	system("ec2-version > /tmp/amazon.log 2>&1");
+	if (!disableAmazon)
+	{
+		system("ec2-version > /tmp/amazon.log 2>&1");
 
-	ifstream in("/tmp/amazon.log");
+		ifstream in("/tmp/amazon.log");
 
-	in.seekg(0, std::ios::end);
-	int size = in.tellg();
-	if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not found")) 
-	// not running on amazon with ec2-api-tools
-		amazonInstall = false;
-	else
-		if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not installed")) 
+		in.seekg(0, std::ios::end);
+		int size = in.tellg();
+		if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not found")) 
+		// not running on amazon with ec2-api-tools
 			amazonInstall = false;
 		else
-			amazonInstall = true;
+			if ( size == 0 || oam.checkLogStatus("/tmp/amazon.log", "not installed")) 
+				amazonInstall = false;
+			else
+				amazonInstall = true;
 
-	string amazonSubNet = oam::UnassignedName;
+		string amazonSubNet = oam::UnassignedName;
+	}
 
 	if ( amazonInstall )
 	{
 		string cloud = oam::UnassignedName;
-		string option = "1";
 
 		try {
 			cloud = sysConfig->getConfig(InstallSection, "Cloud");
@@ -840,42 +847,7 @@ int main(int argc, char *argv[])
 			cloud  = oam::UnassignedName;
 		}
 
-		cout << "===== Amazon EC2-API-TOOLS Instance Install =====" << endl << endl;
-		cout << "You have 2 install options: " << endl << endl;
-		cout << "1. Utilizing the Amazon IDs for instances and volumes which allows for features like" << endl;
-		cout <<     "automaticly launching instances and EBS volumes when configuring and system expansion." << endl;
-		cout <<     "This option is recommended and would be use if you are setting up a MariaDB Columnstore system." << endl << endl;
-		cout << "2. Using standard hardware IDs for hostnames, IP Addresses, and Storage Devices." << endl;
-		cout <<     "Using this option, you would need to pre-create the Instances and the EBS storages" << endl;
-		cout <<     "and then provide the hostnames/IP-Addresses during the configuration and system expansion" << endl;
-		cout <<     "commands. This option would be used when you are installing on a existing system." << endl << endl;
-
-		while(true) {
-			prompt = "Select Install Option [1,2] (" + option + ") > ";
-			pcommand = callReadline(prompt.c_str());
-			if (pcommand) {
-				if (strlen(pcommand) > 0) option = pcommand;
-				callFree(pcommand);
-			}
-	
-			if (option == "2")
-			{
-				amazonInstall = false;
-				break;
-			}
-			else
-			{
-				if ( option != "1" )
-				{
-					cout << "Invalid Entry, please enter '1' or '2'" << endl;
-					if ( noPrompting )
-						exit(1);
-					continue;
-				}
-			}
-
-			cout << endl << "To use the EC2-api-tools, these files will need to be installed on" << endl;
-			cout << "on the local instance:" << endl << endl;
+			cout << endl << "Amazon EC2 Install, these files will need to be installed on the local instance:" << endl << endl;
 			cout << " 1. File containing the Amazon Access Key" << endl;
 			cout << " 2. File containing the Amazon Secret Key" << endl << endl;
 

From 591ff9326fecada2f9e299ef5fa013a875749592 Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Sun, 20 Nov 2016 17:52:57 +0000
Subject: [PATCH 44/54] mcol-404

---
 oamapps/postConfigure/postConfigure.cpp | 37 +++++++++++--------------
 1 file changed, 16 insertions(+), 21 deletions(-)

diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp
index 6862597bd..d036fdea8 100644
--- a/oamapps/postConfigure/postConfigure.cpp
+++ b/oamapps/postConfigure/postConfigure.cpp
@@ -815,6 +815,8 @@ int main(int argc, char *argv[])
 
 	//amazon install setup check
 	bool amazonInstall = false;
+	string amazonSubNet = oam::UnassignedName;
+	string cloud = oam::UnassignedName;
 	if (!disableAmazon)
 	{
 		system("ec2-version > /tmp/amazon.log 2>&1");
@@ -831,14 +833,10 @@ int main(int argc, char *argv[])
 				amazonInstall = false;
 			else
 				amazonInstall = true;
-
-		string amazonSubNet = oam::UnassignedName;
 	}
 
 	if ( amazonInstall )
 	{
-		string cloud = oam::UnassignedName;
-
 		try {
 			cloud = sysConfig->getConfig(InstallSection, "Cloud");
 		}
@@ -847,21 +845,20 @@ int main(int argc, char *argv[])
 			cloud  = oam::UnassignedName;
 		}
 
-			cout << endl << "Amazon EC2 Install, these files will need to be installed on the local instance:" << endl << endl;
-			cout << " 1. File containing the Amazon Access Key" << endl;
-			cout << " 2. File containing the Amazon Secret Key" << endl << endl;
+		cout << endl << "Amazon EC2 Install, these files will need to be installed on the local instance:" << endl << endl;
+		cout << " 1. File containing the Amazon Access Key" << endl;
+		cout << " 2. File containing the Amazon Secret Key" << endl << endl;
 
-			while(true) {
-				string ready = "y";
-				prompt = "Are these files installed and ready to continue [y,n] (y) > ";
-				pcommand = callReadline(prompt.c_str());
-				if (pcommand) {
-					if (strlen(pcommand) > 0) ready = pcommand;
-					callFree(pcommand);
-					if (ready == "n") {
-						cout << endl << "Please Install these files and re-run postConfigure. exiting..." << endl;
-						exit(0);
-					}
+		while(true) {
+			string ready = "y";
+			prompt = "Are these files installed and ready to continue [y,n] (y) > ";
+			pcommand = callReadline(prompt.c_str());
+			if (pcommand) {	
+				if (strlen(pcommand) > 0) ready = pcommand;
+				callFree(pcommand);
+				if (ready == "n") {
+					cout << endl << "Please Install these files and re-run postConfigure. exiting..." << endl;
+					exit(0);
 				}
 
 				if ( ready == "y" )
@@ -948,8 +945,6 @@ int main(int argc, char *argv[])
 			}
 			catch(...)
 			{}
-
-			break;
 		}
 
 		if ( !writeConfig(sysConfig) ) { 
@@ -1045,7 +1040,7 @@ int main(int argc, char *argv[])
 	catch (exception& e) {}
 
 	//get Parent OAM Module Name
-	string parentOAMModuleName = "pm1";
+	parentOAMModuleName = "pm1";
 
 	if ( localModuleName != parentOAMModuleName )
 	{

From 9ce117f521cf8335ebcbc17ddd1b2445b42f8c2a Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Sun, 20 Nov 2016 22:47:54 +0000
Subject: [PATCH 45/54] MCOL-61, ami install

---
 oam/oamcpp/liboamcpp.cpp                |  24 +++-
 oamapps/postConfigure/postConfigure.cpp | 167 +++++++++++++-----------
 2 files changed, 110 insertions(+), 81 deletions(-)

diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp
index 14f2d8038..dbcb1c9da 100644
--- a/oam/oamcpp/liboamcpp.cpp
+++ b/oam/oamcpp/liboamcpp.cpp
@@ -5984,7 +5984,15 @@ namespace oam
 
 		//format attached volume
 		writeLog("addUMdisk - Format new Volume for: " + volumeName, LOG_TYPE_DEBUG);
-		string cmd = "mkfs.ext2 -F " + device + " > /dev/null 2>&1";
+
+		string cmd;
+       	int user;
+        user = getuid();
+        if (user == 0)
+			cmd = "mkfs.ext2 -F " + device + " > /dev/null 2>&1";
+		else
+            cmd = "sudo mkfs.ext2 -F " + device + " > /dev/null 2>&1";
+
 		system(cmd.c_str());
 
 		//detach volume
@@ -6202,7 +6210,13 @@ namespace oam
 			
 				//format attached volume
 				cout << "  Formatting DBRoot #" << itoa(*pt1) << ", please wait..." << endl;
-				string cmd = "mkfs.ext2 -F " + amazonDeviceName + " > /tmp/format.log 2>&1";
+		        string cmd;
+        		int user;
+    		    user = getuid();
+     			if (user == 0)
+            		cmd = "mkfs.ext2 -F " + amazonDeviceName + " > /tmp/format.log 2>&1";
+        		else
+            		cmd = "sudo mkfs.ext2 -F " + amazonDeviceName + " > /tmp/format.log 2>&1";
 
 				writeLog("addDbroot format cmd: " + cmd, LOG_TYPE_DEBUG );
 
@@ -8555,7 +8569,11 @@ namespace oam
 			return "";
 
 		//update /etc/fstab with mount
-		string entry = device + " " + InstallDir + "/data" + dbrootID + " ext2 noatime,nodiratime,noauto 0 0";
+		string entry;
+		if (user == 0)
+			entry = device + " " + InstallDir + "/data" + dbrootID + " ext2 noatime,nodiratime,noauto 0 0";
+		else
+            entry = device + " " + InstallDir + "/data" + dbrootID + " ext2 noatime,nodiratime,noauto,user 0 0";
 
 		//update local fstab
         if (user == 0)
diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp
index d036fdea8..e6492869e 100644
--- a/oamapps/postConfigure/postConfigure.cpp
+++ b/oamapps/postConfigure/postConfigure.cpp
@@ -861,12 +861,12 @@ int main(int argc, char *argv[])
 					exit(0);
 				}
 
-				if ( ready == "y" )
-					break;
-
-				cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl;
-				if ( noPrompting )
-					exit(1);
+				if ( ready != "y" )
+				{
+					cout << "Invalid Entry, please enter 'y' for yes or 'n' for no" << endl;
+					if ( noPrompting )
+						exit(1);
+				}
 			}
 
 			try {
@@ -925,28 +925,30 @@ int main(int argc, char *argv[])
 			if( !copyKeyfiles() )
 				cout << "copyKeyfiles error" << endl;
 
-			try {
-				AmazonRegion = sysConfig->getConfig(InstallSection, "AmazonRegion");
-			}
-			catch(...)
-			{}
-
-			cout << endl;
-
-			prompt = "Enter Amazon Region you are running in (" + AmazonRegion + ") > ";
-			pcommand = callReadline(prompt.c_str());
-			if (pcommand) {
-				if (strlen(pcommand) > 0) AmazonRegion = pcommand;
-				callFree(pcommand);
-			}
-	
-			try {
-				sysConfig->setConfig(InstallSection, "AmazonRegion", AmazonRegion);
-			}
-			catch(...)
-			{}
+			break;
 		}
 
+		try {
+			AmazonRegion = sysConfig->getConfig(InstallSection, "AmazonRegion");
+		}
+		catch(...)
+		{}
+
+		cout << endl;
+
+		prompt = "Enter Amazon Region you are running in (" + AmazonRegion + ") > ";
+		pcommand = callReadline(prompt.c_str());
+		if (pcommand) {
+			if (strlen(pcommand) > 0) AmazonRegion = pcommand;
+			callFree(pcommand);
+		}
+	
+		try {
+			sysConfig->setConfig(InstallSection, "AmazonRegion", AmazonRegion);
+		}
+		catch(...)
+		{}
+
 		if ( !writeConfig(sysConfig) ) { 
 			cout << "ERROR: Failed trying to update MariaDB Columnstore System Configuration file" << endl; 
 			exit(1);
@@ -954,54 +956,51 @@ int main(int argc, char *argv[])
 
 		sleep(1);
 
-		if ( amazonInstall )
-		{
-		      //get subnetID
-		      try {
-			      amazonSubNet = sysConfig->getConfig(InstallSection, "AmazonSubNetID");
-		      }
-		      catch(...)
-		      {}
+		//get subnetID
+	    try {
+		      amazonSubNet = sysConfig->getConfig(InstallSection, "AmazonSubNetID");
+		}
+		catch(...)
+      	{}
 
-		      if ( amazonSubNet == oam::UnassignedName )
-		      {	
-			      //check if this is a vpc system by checking for subnet setup
-			      amazonSubNet = oam.getEC2LocalInstanceSubnet();
-			      // cout << "amazonSubNet = " <<  amazonSubNet << endl;
-			      if ( amazonSubNet == "failed" || amazonSubNet == "" )
-			      {
-				      amazonSubNet = oam::UnassignedName;
-				      cloud = "amazon-ec2";
-			      }
-			      else
-			      {
-				      cloud = "amazon-vpc";
-			      }
+		if ( amazonSubNet == oam::UnassignedName )
+		{	
+			//check if this is a vpc system by checking for subnet setup
+			amazonSubNet = oam.getEC2LocalInstanceSubnet();
+			// cout << "amazonSubNet = " <<  amazonSubNet << endl;
+			if ( amazonSubNet == "failed" || amazonSubNet == "" )
+			{
+				amazonSubNet = oam::UnassignedName;
+				cloud = "amazon-ec2";
+			}
+			else
+			{
+				cloud = "amazon-vpc";
+			}
 
-			      //set subnetID
-			      try {
-				      sysConfig->setConfig(InstallSection, "AmazonSubNetID", amazonSubNet);
-			      }
-			      catch(...)
-			      {}
-		      }
-		      else
-			      cloud = "amazon-vpc";
-
-		      try {
-			      sysConfig->setConfig(InstallSection, "Cloud", cloud);
-		      }
-		      catch(...)
-		      {}
+			//set subnetID
+			try {
+				sysConfig->setConfig(InstallSection, "AmazonSubNetID", amazonSubNet);
+			}
+			catch(...)
+			{}
 		}
 		else
-		{
-		    try {
-			      sysConfig->setConfig(InstallSection, "Cloud", oam::UnassignedName);
-		    }
-		    catch(...)
-		    {}
+			cloud = "amazon-vpc";
+
+		try {
+			sysConfig->setConfig(InstallSection, "Cloud", cloud);
 		}
+		catch(...)
+		{}
+	}
+	else
+	{
+		try {
+			sysConfig->setConfig(InstallSection, "Cloud", oam::UnassignedName);
+		}
+		catch(...)
+		{}
 	}
 
 	if ( pmwithum )
@@ -2711,6 +2710,7 @@ int main(int argc, char *argv[])
 	/* create a thread_data_t argument array */
 	thread_data_t thr_data[childmodulelist.size()];
 
+	string install = "y";
 	if ( IserverTypeInstall != oam::INSTALL_COMBINE_DM_UM_PM || 
 			pmNumber > 1 ) {
 		//
@@ -2718,7 +2718,6 @@ int main(int argc, char *argv[])
 		//
 		cout << endl << "===== System Installation =====" << endl << endl;
 	
-		string install = "y";
 		cout << "System Configuration is complete, System Installation is the next step." << endl;
 
 		while(true)
@@ -2783,13 +2782,22 @@ int main(int argc, char *argv[])
 			}
 
 			if ( EEPackageType == "rpm" )
-				cout << "Performing an MariaDB Columnstore System install using RPM packages located in the " + HOME + " directory." << endl;
+			{
+				cout << "Performing an MariaDB Columnstore System install using RPM packages" << endl; 
+				cout << " located in the " + HOME + " directory." << endl;
+			}
 			else
 			{
 				if ( EEPackageType == "binary" )
-					cout << "Performing an MariaDB Columnstore System install using a Binary package located in the " + HOME + " directory." << endl;
+				{
+					cout << "Performing an MariaDB Columnstore System install using a Binary package" << endl; 
+					cout << "located in the " + HOME + " directory." << endl;
+				}
 				else
-					cout << "Performing an MariaDB Columnstore System install using using DEB packages located in the " + HOME + " directory." << endl;
+				{
+					cout << "Performing an MariaDB Columnstore System install using using DEB packages" << endl;
+					cout << "located in the " + HOME + " directory." << endl;
+				}
 			}
 	
 			//Write out Updated System Configuration File
@@ -3355,16 +3363,19 @@ int main(int argc, char *argv[])
 					string remoteModuleName = (*list1).moduleName;
 					string remoteModuleIP = (*list1).moduleIP;
 					string remoteHostName = (*list1).hostName;
-			
+		
 					//run remote command script
 					cout << endl << "----- Starting MariaDB Columnstore on '" + remoteModuleName + "' -----" << endl << endl;
 
-					cmd = installDir + "/bin/remote_scp_put.sh " + remoteModuleIP + " " + installDir + "/etc/Columnstore.xml  > /dev/null 2>&1";
+					if ( install == "n" ) 
+					{	// didnt do a full install, push the config file
+						cmd = installDir + "/bin/remote_scp_put.sh " + remoteModuleIP + " " + installDir + "/etc/Columnstore.xml  > /dev/null 2>&1";
+						system(cmd.c_str());
+					}
+
+					cmd = installDir + "/bin/remote_command.sh " + remoteModuleIP + " " + password + " '" + installDir + "/bin/columnstore restart' 0";
 					int rtnCode = system(cmd.c_str());
 
-					cmd = installDir + "/bin/remote_command.sh " + remoteModuleIP + " " + password +
-						" '" + installDir + "/bin/columnstore restart' 0";
-					rtnCode = system(cmd.c_str());
 					if (WEXITSTATUS(rtnCode) != 0)
 						cout << "Error with running remote_command.sh" << endl;
 					else

From 606284ad634517e8c59304fc0a740c13015068d4 Mon Sep 17 00:00:00 2001
From: david hill <david.hill@mariadb.com>
Date: Sun, 20 Nov 2016 20:38:32 -0600
Subject: [PATCH 46/54] Update README.md

update release to 1.0.5
---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index da3148b24..b561c70a3 100644
--- a/README.md
+++ b/README.md
@@ -1,9 +1,9 @@
-#MariaDB ColumnStore Storage/Execution engine 1.0.4
-MariaDB ColumnStore 1.0.4 is the development version of MariaDB ColumnStore. 
+#MariaDB ColumnStore Storage/Execution engine 1.0.5
+MariaDB ColumnStore 1.0.5 is the development version of MariaDB ColumnStore. 
 It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely 
 new features not found anywhere else.
 
-#MariaDB ColumnStore 1.0.4 is an Beta release. 
+#MariaDB ColumnStore 1.0.5 is an Beta release. 
 This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 
 series are included in this release. 
 

From a5515d582a184ef50b29a06c41d6658db0438082 Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Mon, 21 Nov 2016 20:26:18 +0000
Subject: [PATCH 47/54] mcol-404 changes

---
 oam/install_scripts/master-rep-columnstore.sh | 4 ----
 oam/install_scripts/post-install              | 8 ++++++--
 oam/install_scripts/pre-uninstall             | 6 +++++-
 oam/install_scripts/slave-rep-columnstore.sh  | 4 ----
 oamapps/postConfigure/postConfigure.cpp       | 4 ++--
 5 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/oam/install_scripts/master-rep-columnstore.sh b/oam/install_scripts/master-rep-columnstore.sh
index 16b956d29..1b943c70f 100644
--- a/oam/install_scripts/master-rep-columnstore.sh
+++ b/oam/install_scripts/master-rep-columnstore.sh
@@ -35,10 +35,6 @@ for arg in "$@"; do
 	fi
 done
 
-if { $PASSWORD == "ssh" } {
-	set PASSWORD ""
-}
-
 test -f $installdir/post/functions && . $installdir/post/functions
 
 repUser="mcsrep"
diff --git a/oam/install_scripts/post-install b/oam/install_scripts/post-install
index 564fd87dc..e24b0dfb5 100755
--- a/oam/install_scripts/post-install
+++ b/oam/install_scripts/post-install
@@ -85,8 +85,12 @@ fi
 sudo ln -s /usr/lib64/perl5/CORE/libperl.so /usr/lib64/libperl.so >/dev/null 2>&1
 
 #setup .bashrc
-eval userhome=~$user
-bashFile=$userhome/.bashrc
+if [ $user = "root" ]; then
+	bashFile=/root/.bashrc
+else
+	bashFile=$HOME/.bashrc
+fi
+
 touch ${bashFile}
 /bin/cp -f ${bashFile} ${bashFile}.columnstoreSave > /dev/null 2>&1
 
diff --git a/oam/install_scripts/pre-uninstall b/oam/install_scripts/pre-uninstall
index d81e9c513..43db84d41 100755
--- a/oam/install_scripts/pre-uninstall
+++ b/oam/install_scripts/pre-uninstall
@@ -33,7 +33,11 @@ for arg in "$@"; do
 	fi
 done
 
-. /home/$user/.bashrc
+if [ $user = "root" ]; then
+	. /root/.bashrc
+else
+	. $HOME/.bashrc
+fi
 
 #stop services
 $installdir/bin/columnstore stop > /dev/null 2>&1
diff --git a/oam/install_scripts/slave-rep-columnstore.sh b/oam/install_scripts/slave-rep-columnstore.sh
index 6054e056b..d56b1cb83 100644
--- a/oam/install_scripts/slave-rep-columnstore.sh
+++ b/oam/install_scripts/slave-rep-columnstore.sh
@@ -41,10 +41,6 @@ for arg in "$@"; do
 	fi
 done
 
-if { $PASSWORD == "ssh" } {
-	set PASSWORD ""
-}
-
 test -f $installdir/post/functions && . $installdir/post/functions
 
 repUser="mscrep"
diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp
index e6492869e..36cf9c762 100644
--- a/oamapps/postConfigure/postConfigure.cpp
+++ b/oamapps/postConfigure/postConfigure.cpp
@@ -352,7 +352,7 @@ int main(int argc, char *argv[])
 				exit (1);
 			}
 		}
-        else if( string("da") == argv[i] )
+        else if( string("-da") == argv[i] )
             disableAmazon = true;
         else if( string("-i") == argv[i] ) {
             i++;
@@ -365,7 +365,7 @@ int main(int argc, char *argv[])
 		else
 		{
 			cout << "   ERROR: Invalid Argument = " << argv[i] << endl;
-   			cout << "   Usage: postConfigure [-h][-c][-u][-p][-mp][-s][-port]" << endl;
+   			cout << "   Usage: postConfigure [-h][-c][-u][-p][-mp][-s][-port][-i][-da]" << endl;
 			exit (1);
 		}
 	}

From 9ad8a3194f2daa04652e157857ed582ca5b8326b Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Tue, 22 Nov 2016 00:29:43 +0000
Subject: [PATCH 48/54] mcol-404

---
 oam/install_scripts/master-rep-columnstore.sh | 3 ++-
 oam/install_scripts/slave-rep-columnstore.sh  | 4 ++--
 procmgr/processmanager.cpp                    | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/oam/install_scripts/master-rep-columnstore.sh b/oam/install_scripts/master-rep-columnstore.sh
index 1b943c70f..a4b0992ed 100644
--- a/oam/install_scripts/master-rep-columnstore.sh
+++ b/oam/install_scripts/master-rep-columnstore.sh
@@ -37,7 +37,8 @@ done
 
 test -f $installdir/post/functions && . $installdir/post/functions
 
-repUser="mcsrep"
+repUser="idbrep"
+password="Calpont1"
 
 >/tmp/master-rep-status-$hostipaddr.log
 
diff --git a/oam/install_scripts/slave-rep-columnstore.sh b/oam/install_scripts/slave-rep-columnstore.sh
index d56b1cb83..efd6609f2 100644
--- a/oam/install_scripts/slave-rep-columnstore.sh
+++ b/oam/install_scripts/slave-rep-columnstore.sh
@@ -43,7 +43,8 @@ done
 
 test -f $installdir/post/functions && . $installdir/post/functions
 
-repUser="mscrep"
+repUser="idbrep"
+password="Calpont1"
 
 >/tmp/slave-rep-status.log
 
@@ -73,7 +74,6 @@ CHANGE MASTER TO
     	MASTER_PORT=$port,
    	MASTER_LOG_FILE='$masterlogfile',
     	MASTER_LOG_POS=$masterlogpos;
-	MASTER_USE_GTID=$masterlogpos;
 
 EOD
 
diff --git a/procmgr/processmanager.cpp b/procmgr/processmanager.cpp
index c56ebfb94..e36827c16 100644
--- a/procmgr/processmanager.cpp
+++ b/procmgr/processmanager.cpp
@@ -4471,7 +4471,7 @@ int ProcessManager::addModule(oam::DeviceNetworkList devicenetworklist, std::str
 		if ( packageType == "deb" )
 			separator = "_";
 		//mariadb
-		calpontPackage = homedir + "/mariadb-columnstore-*" + separator + systemsoftware.Version + "-" + systemsoftware.Release + "*." + packageType;
+		calpontPackage = homedir + "/mariadb-columnstore*" + separator + systemsoftware.Version + "-" + systemsoftware.Release + "*." + packageType;
 		mysqlPackage = homedir + "/mariadb-columnstore-storage-engine" + separator + systemsoftware.Version + "-" + systemsoftware.Release + "*." + packageType;
 		mysqldPackage = homedir + "/mariadb-columnstore-mysql" + separator + systemsoftware.Version + "-" + systemsoftware.Release + "*." + packageType;
 		calpontPackage1 = homedir + "/mariadb-columnstore-libs" + separator + systemsoftware.Version + "-" + systemsoftware.Release + "*." + packageType;

From 3b1de94cd839e4ec2d719784633fd01d42d01bde Mon Sep 17 00:00:00 2001
From: Andrew Hutchings <andrew@linuxjedi.co.uk>
Date: Wed, 23 Nov 2016 22:11:26 +0000
Subject: [PATCH 49/54] MCOL-406 Improved Information Schema

* Add INFORMATION_SCHEMA.COLUMNSTORE_FILES which contains information
  about files
* Remove file information from COLUMNSTORE_EXTENTS (due to above)
* Hide columns with Object ID < 3000 (internal columns)
* Fix bad calculation in data_size columns
* Fix minor memory leak
* Add compressedSize() function to IDBFileSystem to get the used file
  size for a compressed file
* Add columnstore_info schema with utility stored procedures to access
  the information_schema tables
---
 cpackEngineRPM.cmake                         |   4 +
 dbcon/mysql/CMakeLists.txt                   |  15 +-
 dbcon/mysql/columnstore_info.sql             |  50 ++++++
 dbcon/mysql/install_calpont_mysql.sh         |   2 +
 dbcon/mysql/is_columnstore_extents.cpp       |  34 +---
 dbcon/mysql/is_columnstore_files.cpp         | 163 +++++++++++++++++++
 oam/install_scripts/post-mysqld-install      |   3 +
 utils/idbdatafile/IDBFileSystem.h            |   7 +
 utils/idbdatafile/IDBPolicy.h                |   7 +
 utils/idbdatafile/PosixFileSystem.cpp        |  81 +++++++++
 utils/idbdatafile/PosixFileSystem.h          |   1 +
 utils/idbhdfs/hdfs-shared/HdfsFileSystem.cpp |  82 ++++++++++
 utils/idbhdfs/hdfs-shared/HdfsFileSystem.h   |   1 +
 13 files changed, 422 insertions(+), 28 deletions(-)
 create mode 100644 dbcon/mysql/columnstore_info.sql
 create mode 100644 dbcon/mysql/is_columnstore_files.cpp

diff --git a/cpackEngineRPM.cmake b/cpackEngineRPM.cmake
index d26a1d98f..32552e855 100644
--- a/cpackEngineRPM.cmake
+++ b/cpackEngineRPM.cmake
@@ -327,6 +327,9 @@ SET(CPACK_RPM_storage-engine_USER_FILELIST
 "/usr/local/mariadb/columnstore/lib/is_columnstore_tables.so"
 "/usr/local/mariadb/columnstore/lib/is_columnstore_tables.so.1"
 "/usr/local/mariadb/columnstore/lib/is_columnstore_tables.so.1.0.0"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_files.so"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_files.so.1"
+"/usr/local/mariadb/columnstore/lib/is_columnstore_files.so.1.0.0"
 "/usr/local/mariadb/columnstore/mysql/mysql-Columnstore"
 "/usr/local/mariadb/columnstore/mysql/install_calpont_mysql.sh"
 "/usr/local/mariadb/columnstore/mysql/syscatalog_mysql.sql"
@@ -335,6 +338,7 @@ SET(CPACK_RPM_storage-engine_USER_FILELIST
 "/usr/local/mariadb/columnstore/mysql/calsetuserpriority.sql"
 "/usr/local/mariadb/columnstore/mysql/calremoveuserpriority.sql"
 "/usr/local/mariadb/columnstore/mysql/calshowprocesslist.sql"
+"/usr/local/mariadb/columnstore/mysql/columnstore_info.sql"
 ${ignored})
 
 
diff --git a/dbcon/mysql/CMakeLists.txt b/dbcon/mysql/CMakeLists.txt
index 7d9b012ca..d478843b0 100644
--- a/dbcon/mysql/CMakeLists.txt
+++ b/dbcon/mysql/CMakeLists.txt
@@ -62,12 +62,25 @@ target_link_libraries(is_columnstore_extents ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LI
 set_target_properties(is_columnstore_extents PROPERTIES PREFIX "")
 set_target_properties(is_columnstore_extents PROPERTIES VERSION 1.0.0 SOVERSION 1)
 
-install(TARGETS calmysql is_columnstore_tables is_columnstore_columns is_columnstore_extents DESTINATION ${ENGINE_LIBDIR} COMPONENT storage-engine)
+SET ( is_columnstore_files_SRCS
+    is_columnstore_files.cpp
+    )
+add_library(is_columnstore_files SHARED ${is_columnstore_files_SRCS})
+
+target_link_libraries(is_columnstore_files ${ENGINE_LDFLAGS} ${ENGINE_WRITE_LIBS} threadpool)
+
+# Don't prepend .so file with 'lib'
+set_target_properties(is_columnstore_files PROPERTIES PREFIX "")
+set_target_properties(is_columnstore_files PROPERTIES VERSION 1.0.0 SOVERSION 1)
+
+
+install(TARGETS calmysql is_columnstore_tables is_columnstore_columns is_columnstore_extents is_columnstore_files DESTINATION ${ENGINE_LIBDIR} COMPONENT storage-engine)
 install(FILES syscatalog_mysql.sql 
               dumpcat_mysql.sql 
               calsetuserpriority.sql 
               calremoveuserpriority.sql 
               calshowprocesslist.sql 
+              columnstore_info.sql
               my.cnf
               DESTINATION ${ENGINE_MYSQLDIR} COMPONENT storage-engine)
 install(PROGRAMS install_calpont_mysql.sh mysql-Columnstore dumpcat.pl
diff --git a/dbcon/mysql/columnstore_info.sql b/dbcon/mysql/columnstore_info.sql
new file mode 100644
index 000000000..b545ce647
--- /dev/null
+++ b/dbcon/mysql/columnstore_info.sql
@@ -0,0 +1,50 @@
+CREATE DATABASE columnstore_info;
+USE columnstore_info;
+
+DELIMITER //
+CREATE FUNCTION `format_filesize`(filesize FLOAT) RETURNS varchar(20) CHARSET utf8 DETERMINISTIC
+BEGIN
+
+DECLARE n INT DEFAULT 1;
+
+LOOP
+    IF filesize < 1024 THEN
+        RETURN concat(round(filesize, 2), ' ', elt(n, 'Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB', 'BB'));
+    END IF;
+    SET filesize = filesize / 1024;
+    SET n = n + 1;
+END LOOP;
+
+END //
+
+
+CREATE PROCEDURE total_usage ()
+BEGIN
+    SELECT
+        (SELECT format_filesize(sum(data_size)) TOTAL_DATA_SIZE FROM INFORMATION_SCHEMA.COLUMNSTORE_EXTENTS) TOTAL_DATA_SIZE,
+        (SELECT format_filesize(sum(file_size)) TOTAL_DISK_USAGE FROM INFORMATION_SCHEMA.COLUMNSTORE_FILES) TOTAL_DISK_USAGE;
+END //
+
+CREATE PROCEDURE table_usage (IN t_name char(64))
+BEGIN
+    IF t_name IS NOT NULL THEN
+    SELECT TABLE_NAME, format_filesize(sum(cf.file_size)) DATA_DISK_USAGE, format_filesize(sum(IFNULL(ccf.file_size, 0))) DICT_DISK_USAGE, format_filesize(sum(cf.file_size) + sum(IFNULL(ccf.file_size, 0))) TOTAL_USAGE FROM INFORMATION_SCHEMA.COLUMNSTORE_COLUMNS cc
+JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES cf ON cc.object_id = cf.object_id
+LEFT JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES ccf ON cc.dictionary_object_id = ccf.object_id
+WHERE table_name = t_name GROUP BY table_name;
+    ELSE
+    SELECT TABLE_NAME, format_filesize(sum(cf.file_size)) DATA_DISK_USAGE, format_filesize(sum(IFNULL(ccf.file_size, 0))) DICT_DISK_USAGE, format_filesize(sum(cf.file_size) + sum(IFNULL(ccf.file_size, 0))) TOTAL_USAGE FROM INFORMATION_SCHEMA.COLUMNSTORE_COLUMNS cc
+JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES cf ON cc.object_id = cf.object_id
+LEFT JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES ccf ON cc.dictionary_object_id = ccf.object_id
+GROUP BY table_name;
+    END IF;
+END //
+    
+CREATE PROCEDURE compression_ratio()
+BEGIN
+SELECT CONCAT(((1 - (sum(data_size) / sum(compressed_data_size))) * 100), '%') COMPRESSION_RATIO FROM INFORMATION_SCHEMA.COLUMNSTORE_EXTENTS ce
+JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES cf ON ce.object_id = cf.object_id
+WHERE compressed_data_size IS NOT NULL;
+END //
+
+DELIMITER ;
diff --git a/dbcon/mysql/install_calpont_mysql.sh b/dbcon/mysql/install_calpont_mysql.sh
index d04bc3487..9b55f9448 100755
--- a/dbcon/mysql/install_calpont_mysql.sh
+++ b/dbcon/mysql/install_calpont_mysql.sh
@@ -33,6 +33,7 @@ INSTALL PLUGIN infinidb SONAME 'libcalmysql.so';
 INSTALL PLUGIN columnstore_tables SONAME 'is_columnstore_tables.so';
 INSTALL PLUGIN columnstore_columns SONAME 'is_columnstore_columns.so';
 INSTALL PLUGIN columnstore_extents SONAME 'is_columnstore_extents.so';
+INSTALL PLUGIN columnstore_files SONAME 'is_columnstore_files.so';
 -- these are deprecated names
 DELETE FROM mysql.func WHERE name='caldisablepartition';
 DELETE FROM mysql.func WHERE name='caldroppartition';
@@ -133,6 +134,7 @@ $installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/d
 $installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/dev/null <$installdir/mysql/calsetuserpriority.sql
 $installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/dev/null <$installdir/mysql/calremoveuserpriority.sql
 $installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/dev/null <$installdir/mysql/calshowprocesslist.sql
+$installdir/mysql/bin/mysql --defaults-file=$df --user=root $pwprompt mysql 2>/dev/null <$installdir/mysql/columnstore_info.sql
 
 sed -i 's/infinidb_compression_type=1/infinidb_compression_type=2/' $installdir/mysql/my.cnf >/dev/null 2>&1
 
diff --git a/dbcon/mysql/is_columnstore_extents.cpp b/dbcon/mysql/is_columnstore_extents.cpp
index fa5835888..1d1098eda 100644
--- a/dbcon/mysql/is_columnstore_extents.cpp
+++ b/dbcon/mysql/is_columnstore_extents.cpp
@@ -24,13 +24,9 @@
 #include <vector>
 #include <limits>
 
-#include <boost/shared_ptr.hpp>
 #include "dbrm.h"
 #include "objectidmanager.h"
-#include "we_convertor.h"
-#include "we_define.h"
-#include "IDBPolicy.h"
-#include "we_config.h"
+
 
 // Required declaration as it isn't in a MairaDB include
 bool schema_table_store_record(THD *thd, TABLE *table);
@@ -53,8 +49,6 @@ ST_FIELD_INFO is_columnstore_extents_fields[] =
     {"STATE", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},                       // 13
     {"STATUS", 64, MYSQL_TYPE_STRING, 0, 0, 0, 0},                      // 14
     {"DATA_SIZE", 19, MYSQL_TYPE_LONGLONG, 0, 0, 0, 0},                 // 15
-    {"FILENAME", 1024, MYSQL_TYPE_STRING, 0, 0, 0, 0},                  // 16
-    {"FILE_SIZE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0}, // 17
     {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0}
 };
 
@@ -65,9 +59,6 @@ static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond)
     std::vector<struct BRM::EMEntry> entries;
    	std::vector<struct BRM::EMEntry>::iterator iter;
 	std::vector<struct BRM::EMEntry>::iterator end;
-	char oidDirName[WriteEngine::FILE_NAME_SIZE];
-	char fullFileName[WriteEngine::FILE_NAME_SIZE];
-    char dbDir[WriteEngine::MAX_DB_DIR_LEVEL][WriteEngine::MAX_DB_DIR_NAME_SIZE];
 
     BRM::DBRM *emp = new BRM::DBRM();
     if (!emp || !emp->isDBRMReady())
@@ -78,7 +69,7 @@ static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond)
     execplan::ObjectIDManager oidm;
     BRM::OID_t MaxOID = oidm.size();
 
-    for(BRM::OID_t oid = 0; oid <= MaxOID; oid++)
+    for(BRM::OID_t oid = 3000; oid <= MaxOID; oid++)
     {
         emp->getExtents(oid, entries, false, false, true);
         if (entries.size() == 0)
@@ -161,31 +152,20 @@ static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond)
                 default:
                     table->field[14]->store("Unknown", strlen("Unknown"), cs);
             }
-            table->field[15]->store((iter->HWM + 1) * 8192);
-
-            WriteEngine::Convertor::oid2FileName(oid, oidDirName, dbDir, iter->partitionNum, iter->segmentNum);
-            snprintf(fullFileName, WriteEngine::FILE_NAME_SIZE, "%s/%s", WriteEngine::Config::getDBRootByNum(iter->dbRoot).c_str(), oidDirName);
-            table->field[16]->store(fullFileName, strlen(fullFileName), cs);
-
-            if (idbdatafile::IDBPolicy::exists(fullFileName))
-            {
-                table->field[17]->set_notnull();
-                table->field[17]->store(idbdatafile::IDBPolicy::size(fullFileName));
-            }
-            else
-            {
-                table->field[17]->set_null();
-            }
+            table->field[15]->store((iter->HWM - iter->blockOffset + 1) * 8192);
 
             if (schema_table_store_record(thd, table))
+            {
+                delete emp;
                 return 1;
+            }
 
             iter++;
 
         }
     }
 
-
+    delete emp;
     return 0;
 }
 
diff --git a/dbcon/mysql/is_columnstore_files.cpp b/dbcon/mysql/is_columnstore_files.cpp
new file mode 100644
index 000000000..b39be0ee6
--- /dev/null
+++ b/dbcon/mysql/is_columnstore_files.cpp
@@ -0,0 +1,163 @@
+/* c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil
+ * vi: set shiftwidth=4 tabstop=4 expandtab:
+ *  :indentSize=4:tabSize=4:noTabs=true:
+ *
+ * Copyright (C) 2016 MariaDB Corporaton
+ *
+ * This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; version 2 of
+ * the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301, USA.
+ */
+
+#include "idb_mysql.h"
+
+#include "dbrm.h"
+#include "objectidmanager.h"
+#include "we_convertor.h"
+#include "we_define.h"
+#include "IDBPolicy.h"
+#include "we_config.h"
+#include "we_brm.h"
+
+// Required declaration as it isn't in a MairaDB include
+bool schema_table_store_record(THD *thd, TABLE *table);
+
+ST_FIELD_INFO is_columnstore_files_fields[] =
+{
+    {"OBJECT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"SEGMENT_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"PARTITION_ID", 11, MYSQL_TYPE_LONG, 0, 0, 0, 0},
+    {"FILENAME", 1024, MYSQL_TYPE_STRING, 0, 0, 0, 0},
+    {"FILE_SIZE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0},
+    {"COMPRESSED_DATA_SIZE", 19, MYSQL_TYPE_LONGLONG, 0, MY_I_S_MAYBE_NULL, 0, 0},
+    {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0}
+};
+
+
+static bool is_columnstore_files_get_entries(THD *thd, TABLE_LIST *tables, BRM::OID_t oid, std::vector<struct BRM::EMEntry> &entries)
+{
+    CHARSET_INFO *cs = system_charset_info;
+    TABLE *table = tables->table;
+
+    char oidDirName[WriteEngine::FILE_NAME_SIZE];
+	char fullFileName[WriteEngine::FILE_NAME_SIZE];
+    char dbDir[WriteEngine::MAX_DB_DIR_LEVEL][WriteEngine::MAX_DB_DIR_NAME_SIZE];
+    WriteEngine::Config config;
+	config.initConfigCache();
+
+    std::vector<struct BRM::EMEntry>::const_iterator iter = entries.begin();
+    while ( iter != entries.end() ) //organize extents into files
+    {
+        // Don't include files more than once at different block offsets
+        if (iter->blockOffset > 0)
+        {
+            iter++;
+            continue;
+        }
+        table->field[0]->store(oid);
+        table->field[1]->store(iter->segmentNum);
+        table->field[2]->store(iter->partitionNum);
+
+        WriteEngine::Convertor::oid2FileName(oid, oidDirName, dbDir, iter->partitionNum, iter->segmentNum);
+        snprintf(fullFileName, WriteEngine::FILE_NAME_SIZE, "%s/%s", WriteEngine::Config::getDBRootByNum(iter->dbRoot).c_str(), oidDirName);
+        table->field[3]->store(fullFileName, strlen(fullFileName), cs);
+
+        if (idbdatafile::IDBPolicy::exists(fullFileName))
+        {
+            table->field[4]->set_notnull();
+            table->field[4]->store(idbdatafile::IDBPolicy::size(fullFileName));
+            off64_t comp_size;
+            if ((comp_size = idbdatafile::IDBPolicy::compressedSize(fullFileName)) != -1)
+            {
+                table->field[5]->set_notnull();
+                table->field[5]->store(comp_size);
+            }
+            else
+            {
+                table->field[5]->set_null();
+            }
+        }
+        else
+        {
+            table->field[4]->set_null();
+            table->field[5]->set_null();
+        }
+
+        if (schema_table_store_record(thd, table))
+            return 1;
+        iter++;
+    }
+    return 0;
+}
+
+static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond)
+{
+    BRM::DBRM *emp = new BRM::DBRM();
+    std::vector<struct BRM::EMEntry> entries;
+
+    if (!emp || !emp->isDBRMReady())
+    {
+        return 1;
+    }
+
+    execplan::ObjectIDManager oidm;
+    BRM::OID_t MaxOID = oidm.size();
+
+    for(BRM::OID_t oid = 3000; oid <= MaxOID; oid++)
+    {
+        emp->getExtents(oid, entries, false, false, true);
+        if (entries.size() == 0)
+            continue;
+
+        if (is_columnstore_files_get_entries(thd, tables, oid, entries))
+        {
+            delete emp;
+            return 1;
+        }
+
+    }
+
+    delete emp;
+    return 0;
+}
+
+static int is_columnstore_files_plugin_init(void *p)
+{
+    ST_SCHEMA_TABLE *schema = (ST_SCHEMA_TABLE*) p;
+    schema->fields_info = is_columnstore_files_fields;
+    schema->fill_table = is_columnstore_files_fill;
+    return 0;
+}
+
+static struct st_mysql_information_schema is_columnstore_files_plugin_version =
+{ MYSQL_INFORMATION_SCHEMA_INTERFACE_VERSION };
+
+maria_declare_plugin(is_columnstore_files_plugin)
+{
+    MYSQL_INFORMATION_SCHEMA_PLUGIN,
+    &is_columnstore_files_plugin_version,
+    "COLUMNSTORE_FILES",
+    "MariaDB Corporaton",
+    "An information schema plugin to list ColumnStore filess",
+    PLUGIN_LICENSE_GPL,
+    is_columnstore_files_plugin_init,
+    //is_columnstore_files_plugin_deinit,
+    NULL,
+    0x0100,
+    NULL,
+    NULL,
+    "1.0",
+    MariaDB_PLUGIN_MATURITY_STABLE
+}
+maria_declare_plugin_end;
diff --git a/oam/install_scripts/post-mysqld-install b/oam/install_scripts/post-mysqld-install
index 92ea035fa..546945b22 100755
--- a/oam/install_scripts/post-mysqld-install
+++ b/oam/install_scripts/post-mysqld-install
@@ -85,12 +85,14 @@ if [ -f $installdir/lib/libcalmysql.so.1.0.0 ]; then
     is_columnstore_tables=$installdir/lib/is_columnstore_tables.so.1.0.0
     is_columnstore_columns=$installdir/lib/is_columnstore_columns.so.1.0.0
     is_columnstore_extents=$installdir/lib/is_columnstore_extents.so.1.0.0
+    is_columnstore_files=$installdir/lib/is_columnstore_files.so.1.0.0
 elif [ -f $installdir/lib/libcalmysql.so.1 ]; then
 	libcalmysql=$installdir/lib/libcalmysql.so.1
 	libudfsdk=$installdir/lib/libudf_mysql.so.1
     is_columnstore_tables=$installdir/lib/is_columnstore_tables.so.1
     is_columnstore_columns=$installdir/lib/is_columnstore_columns.so.1
     is_columnstore_extents=$installdir/lib/is_columnstore_extents.so.1
+    is_columnstore_files=$installdir/lib/is_columnstore_files.so.1
 else
 	libcalmysql=
 fi
@@ -104,6 +106,7 @@ if [ -n "$libcalmysql" ]; then
     ln -sf $is_columnstore_tables is_columnstore_tables.so
     ln -sf $is_columnstore_columns is_columnstore_columns.so
     ln -sf $is_columnstore_extents is_columnstore_extents.so
+    ln -sf $is_columnstore_files is_columnstore_files.so
 fi
 
 # cleanup previous install mysql replication files
diff --git a/utils/idbdatafile/IDBFileSystem.h b/utils/idbdatafile/IDBFileSystem.h
index 2b5c0afbf..d0cc55613 100644
--- a/utils/idbdatafile/IDBFileSystem.h
+++ b/utils/idbdatafile/IDBFileSystem.h
@@ -84,6 +84,13 @@ public:
 	 */
 	virtual off64_t size(const char* path) const = 0;
 
+    /**
+     * compressedSize() returns the decompressed size of the file
+     * speicified by path.
+     * Returns the size on success, -1 on error
+     */
+    virtual off64_t compressedSize(const char* path) const = 0;
+
 	/**
 	 * exists() checks for the existence of a particular path.
 	 * Returns true if exists, false otherwise.
diff --git a/utils/idbdatafile/IDBPolicy.h b/utils/idbdatafile/IDBPolicy.h
index 81891104c..433bde726 100644
--- a/utils/idbdatafile/IDBPolicy.h
+++ b/utils/idbdatafile/IDBPolicy.h
@@ -115,6 +115,7 @@ public:
 	 */
 	static int mkdir(const char *pathname);
 	static off64_t size(const char* path);
+    static off64_t compressedSize(const char* path);
 	static int remove(const char *pathname);
 	static int rename(const char *oldpath, const char *newpath);
 	static bool exists(const char* pathname);
@@ -175,6 +176,12 @@ off64_t IDBPolicy::size(const char* path)
 	return IDBPolicy::getFs( path ).size( path );
 }
 
+inline
+off64_t IDBPolicy::compressedSize(const char* path)
+{
+    return IDBPolicy::getFs( path ).compressedSize( path );
+}
+
 inline
 int IDBPolicy::remove(const char *pathname)
 {
diff --git a/utils/idbdatafile/PosixFileSystem.cpp b/utils/idbdatafile/PosixFileSystem.cpp
index 38822d2fa..59685c1f9 100644
--- a/utils/idbdatafile/PosixFileSystem.cpp
+++ b/utils/idbdatafile/PosixFileSystem.cpp
@@ -17,6 +17,7 @@
 
 #include "PosixFileSystem.h"
 #include "IDBLogger.h"
+#include "idbcompress.h"
 
 #include <sys/stat.h>
 #include <unistd.h>
@@ -128,6 +129,86 @@ off64_t PosixFileSystem::size(const char* path) const
 	return ret;
 }
 
+size_t readFillBuffer(
+	idbdatafile::IDBDataFile* pFile,
+	char*   buffer,
+	size_t  bytesReq)
+{
+	char*   pBuf = buffer;
+	ssize_t nBytes;
+	size_t  bytesToRead = bytesReq;
+	size_t  totalBytesRead = 0;
+
+	while (1)
+	{
+		nBytes = pFile->read(pBuf, bytesToRead);
+		if (nBytes > 0)
+			totalBytesRead += nBytes;
+		else
+			break;
+
+		if ((size_t)nBytes == bytesToRead)
+			break;
+
+		pBuf        += nBytes;
+		bytesToRead  =  bytesToRead - (size_t)nBytes;
+	}
+
+	return totalBytesRead;
+}
+
+off64_t PosixFileSystem::compressedSize(const char *path) const
+{
+    IDBDataFile *pFile = 0;
+    size_t nBytes;
+    off64_t dataSize = 0;
+
+    try
+    {
+        pFile = IDBDataFile::open(IDBDataFile::BUFFERED, path, "r", 0);
+
+        if (!pFile)
+        {
+            return -1;
+        }
+
+        compress::IDBCompressInterface decompressor;
+
+        char hdr1[compress::IDBCompressInterface::HDR_BUF_LEN];
+        nBytes = readFillBuffer( pFile,hdr1,compress::IDBCompressInterface::HDR_BUF_LEN);
+        if ( nBytes != compress::IDBCompressInterface::HDR_BUF_LEN )
+        {
+            return -1;
+        }
+
+        int64_t ptrSecSize = decompressor.getHdrSize(hdr1) - compress::IDBCompressInterface::HDR_BUF_LEN;
+        char* hdr2 = new char[ptrSecSize];
+        nBytes = readFillBuffer( pFile,hdr2,ptrSecSize);
+        if ( (int64_t)nBytes != ptrSecSize )
+        {
+            return -1;
+        }
+
+        compress::CompChunkPtrList chunkPtrs;
+        int rc = decompressor.getPtrList(hdr2, ptrSecSize, chunkPtrs);
+        delete[] hdr2;
+        if (rc != 0)
+        {
+            return -1;
+        }
+
+        unsigned k = chunkPtrs.size();
+        // last header's offset + length will be the data bytes
+        dataSize = chunkPtrs[k-1].first + chunkPtrs[k-1].second;
+        delete pFile;
+        return dataSize;
+    }
+    catch (...)
+    {
+        return -1;
+    }
+}
+
 bool PosixFileSystem::exists(const char *pathname) const
 {
 	boost::filesystem::path dirPath(pathname);
diff --git a/utils/idbdatafile/PosixFileSystem.h b/utils/idbdatafile/PosixFileSystem.h
index c5c53d162..9c22f805e 100644
--- a/utils/idbdatafile/PosixFileSystem.h
+++ b/utils/idbdatafile/PosixFileSystem.h
@@ -31,6 +31,7 @@ public:
 
 	/* virtual */ int mkdir(const char *pathname);
 	/* virtual */ off64_t size(const char* path) const;
+    /* virtual */ off64_t compressedSize(const char *path) const;
 	/* virtual */ int remove(const char *pathname);
 	/* virtual */ int rename(const char *oldpath, const char *newpath);
 	/* virtual */ bool exists(const char* pathname) const;
diff --git a/utils/idbhdfs/hdfs-shared/HdfsFileSystem.cpp b/utils/idbhdfs/hdfs-shared/HdfsFileSystem.cpp
index 11ef8bee9..1e5626147 100644
--- a/utils/idbhdfs/hdfs-shared/HdfsFileSystem.cpp
+++ b/utils/idbhdfs/hdfs-shared/HdfsFileSystem.cpp
@@ -31,6 +31,7 @@
 
 #include "HdfsFsCache.h"
 #include "IDBLogger.h"
+#include "idbcompress.h"
 
 #include <iostream>
 #include <list>
@@ -119,6 +120,87 @@ off64_t HdfsFileSystem::size(const char* path) const
 	return retval;
 }
 
+size_t readFillBuffer(
+	idbdatafile::IDBDataFile* pFile,
+	char*   buffer,
+	size_t  bytesReq)
+{
+	char*   pBuf = buffer;
+	ssize_t nBytes;
+	size_t  bytesToRead = bytesReq;
+	size_t  totalBytesRead = 0;
+
+	while (1)
+	{
+		nBytes = pFile->read(pBuf, bytesToRead);
+		if (nBytes > 0)
+			totalBytesRead += nBytes;
+		else
+			break;
+
+		if ((size_t)nBytes == bytesToRead)
+			break;
+
+		pBuf        += nBytes;
+		bytesToRead  =  bytesToRead - (size_t)nBytes;
+	}
+
+	return totalBytesRead;
+}
+
+
+off64_t HdfsFileSystem::compressedSize(const char *path) const
+{
+    IDBDataFile *pFile = 0;
+    size_t nBytes;
+    off64_t dataSize = 0;
+
+    try
+    {
+        pFile = IDBDataFile::open(IDBDataFile::HDFS, path, "r", 0);
+
+        if (!pFile)
+        {
+            return -1;
+        }
+
+        compress::IDBCompressInterface decompressor;
+
+        char hdr1[compress::IDBCompressInterface::HDR_BUF_LEN];
+        nBytes = readFillBuffer( pFile,hdr1,compress::IDBCompressInterface::HDR_BUF_LEN);
+        if ( nBytes != compress::IDBCompressInterface::HDR_BUF_LEN )
+        {
+            return -1;
+        }
+
+        int64_t ptrSecSize = decompressor.getHdrSize(hdr1) - compress::IDBCompressInterface::HDR_BUF_LEN;
+        char* hdr2 = new char[ptrSecSize];
+        nBytes = readFillBuffer( pFile,hdr2,ptrSecSize);
+        if ( (int64_t)nBytes != ptrSecSize )
+        {
+            return -1;
+        }
+
+        compress::CompChunkPtrList chunkPtrs;
+        int rc = decompressor.getPtrList(hdr2, ptrSecSize, chunkPtrs);
+        delete[] hdr2;
+        if (rc != 0)
+        {
+            return -1;
+        }
+
+        unsigned k = chunkPtrs.size();
+        // last header's offset + length will be the data bytes
+        dataSize = chunkPtrs[k-1].first + chunkPtrs[k-1].second;
+        delete pFile;
+        return dataSize;
+    }
+    catch (...)
+    {
+        return -1;
+    }
+}
+
 bool HdfsFileSystem::exists(const char *pathname) const
 {
 	int ret = hdfsExists(m_fs,pathname);
diff --git a/utils/idbhdfs/hdfs-shared/HdfsFileSystem.h b/utils/idbhdfs/hdfs-shared/HdfsFileSystem.h
index 37c8a58c9..f0a80abe1 100644
--- a/utils/idbhdfs/hdfs-shared/HdfsFileSystem.h
+++ b/utils/idbhdfs/hdfs-shared/HdfsFileSystem.h
@@ -45,6 +45,7 @@ public:
 
 	/* virtual */ int mkdir(const char *pathname);
 	/* virtual */ off64_t size(const char* path) const;
+    /* virtual */ off64_t compressedSize(const char* path) const;
 	/* virtual */ int remove(const char *pathname);
 	/* virtual */ int rename(const char *oldpath, const char *newpath);
 	/* virtual */ bool exists(const char* pathname) const;

From 97b84159b8cba5a3a2eb9cf0b1fc513c38d11016 Mon Sep 17 00:00:00 2001
From: David Hill <david.hill@mariadb.com>
Date: Thu, 24 Nov 2016 15:06:23 +0000
Subject: [PATCH 50/54] MCOL-416

---
 oam/oamcpp/liboamcpp.cpp | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/oam/oamcpp/liboamcpp.cpp b/oam/oamcpp/liboamcpp.cpp
index dbcb1c9da..19ed72529 100644
--- a/oam/oamcpp/liboamcpp.cpp
+++ b/oam/oamcpp/liboamcpp.cpp
@@ -1367,13 +1367,13 @@ namespace oam
 				{
 					processor.shutdown();
 					string error = e.what();
-					writeLog("getSystemStatus: write exception: " + error, LOG_TYPE_ERROR);
+					//writeLog("getSystemStatus: write exception: " + error, LOG_TYPE_ERROR);
 					exceptionControl("getSystemStatus write", API_FAILURE);
 				}
 				catch(...)
 				{
 					processor.shutdown();
-					writeLog("getSystemStatus: write exception: unknown", LOG_TYPE_ERROR);
+					//writeLog("getSystemStatus: write exception: unknown", LOG_TYPE_ERROR);
 					exceptionControl("getSystemStatus write", API_FAILURE);
 				}
 
@@ -1386,13 +1386,13 @@ namespace oam
 				{
 					processor.shutdown();
 					string error = e.what();
-					writeLog("getSystemStatus: read exception: " + error, LOG_TYPE_ERROR);
+					//writeLog("getSystemStatus: read exception: " + error, LOG_TYPE_ERROR);
 					exceptionControl("getSystemStatus read", API_FAILURE);
 				}
 				catch(...)
 				{
 					processor.shutdown();
-					writeLog("getSystemStatus: read exception: unknown", LOG_TYPE_ERROR);
+					//writeLog("getSystemStatus: read exception: unknown", LOG_TYPE_ERROR);
 					exceptionControl("getSystemStatus read", API_FAILURE);
 				}
 
@@ -1475,21 +1475,21 @@ namespace oam
 				}
 				else
 				{
-					writeLog("getSystemStatus: ProcStatusControl returns 0 length", LOG_TYPE_ERROR);
+					//writeLog("getSystemStatus: ProcStatusControl returns 0 length", LOG_TYPE_ERROR);
 				}
 				// timeout ocurred, shutdown connection
 				processor.shutdown();
-				writeLog("getSystemStatus: read 0 length", LOG_TYPE_ERROR);
+				//writeLog("getSystemStatus: read 0 length", LOG_TYPE_ERROR);
 				exceptionControl("getSystemStatus read 0", API_FAILURE);
 			}
 			catch (exception& e)
 			{
 				string error = e.what();
-				writeLog("getSystemStatus: final exception: " + error, LOG_TYPE_ERROR);
+				//writeLog("getSystemStatus: final exception: " + error, LOG_TYPE_ERROR);
 			}
 			catch(...)
 			{
-				writeLog("getSystemStatus: final exception: unknown", LOG_TYPE_ERROR);
+				//writeLog("getSystemStatus: final exception: unknown", LOG_TYPE_ERROR);
 			}
 		}
 
@@ -1564,13 +1564,13 @@ namespace oam
 								Oam oam;
 								ostringstream os;
 								os << "Oam::getModuleStatus exception while getNICStatus " << (*pt1).HostName << " " << e.what();
-								oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
+								//oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
 							}
 							catch (...) {
 								Oam oam;
 								ostringstream os;
 								os << "Oam::getModuleStatus exception while getNICStatus " << (*pt1).HostName;
-								oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
+								//oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
 							}
 						}
 						
@@ -1589,13 +1589,13 @@ namespace oam
 						Oam oam;
 						ostringstream os;
 						os << "Oam::getModuleStatus exception while getSystemConfig " << name << " " << e.what();
-						oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
+						//oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
 					}
 					catch (...) {
 						Oam oam;
 						ostringstream os;
 						os << "Oam::getModuleStatus exception while getSystemConfig " << name;
-						oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
+						//oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
 					}
 				}
 			}
@@ -1605,13 +1605,13 @@ namespace oam
 			Oam oam;
 			ostringstream os;
 			os << "Oam::getModuleStatus exception while getSystemStatus " << e.what();
-			oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
+			//oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
 		}
 		catch (...) {
 			Oam oam;
 			ostringstream os;
 			os << "Oam::getModuleStatus exception while getSystemStatus";
-			oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
+			//oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
 		}
 
         // no match found
@@ -1795,7 +1795,7 @@ namespace oam
 			Oam oam;
 			ostringstream os;
 			os << "Oam::getNICStatus exception while getSystemStatus for " << name << " " << e.what();
-			oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
+			//oam.writeLog(os.str(), logging::LOG_TYPE_ERROR);
 	        exceptionControl("getNICStatus", API_FAILURE);
 		}
 

From c5350c38ce4f02898e88cb6e26ec63d2c91152b2 Mon Sep 17 00:00:00 2001
From: david hill <david.hill@mariadb.com>
Date: Thu, 24 Nov 2016 09:14:36 -0600
Subject: [PATCH 51/54] Update README.md

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index b561c70a3..62a8a218d 100644
--- a/README.md
+++ b/README.md
@@ -1,9 +1,9 @@
-#MariaDB ColumnStore Storage/Execution engine 1.0.5
-MariaDB ColumnStore 1.0.5 is the development version of MariaDB ColumnStore. 
+#MariaDB ColumnStore Storage/Execution engine 1.0.6
+MariaDB ColumnStore 1.0.6 is the development version of MariaDB ColumnStore. 
 It is built by porting InfiniDB 4.6.2 on MariaDB 10.1.19 and adding entirely 
 new features not found anywhere else.
 
-#MariaDB ColumnStore 1.0.5 is an Beta release. 
+#MariaDB ColumnStore 1.0.6 is an GA release. 
 This is the first MariaDB ColumnStore release, not all features planned for the MariaDB ColumnStore 1.0 
 series are included in this release. 
 

From a66870ef43d287cd2a3e83a22e805fdc8402d75f Mon Sep 17 00:00:00 2001
From: Andrew Hutchings <andrew@linuxjedi.co.uk>
Date: Thu, 24 Nov 2016 19:39:11 +0000
Subject: [PATCH 52/54] MCOL-420 Add cpimport to columnstoreAlias

---
 oam/install_scripts/columnstoreAlias | 1 +
 1 file changed, 1 insertion(+)

diff --git a/oam/install_scripts/columnstoreAlias b/oam/install_scripts/columnstoreAlias
index 2cf611819..b1dc3dd9c 100644
--- a/oam/install_scripts/columnstoreAlias
+++ b/oam/install_scripts/columnstoreAlias
@@ -3,6 +3,7 @@
 alias mcsmysql='/usr/local/mariadb/columnstore/mysql/bin/mysql --defaults-file=/usr/local/mariadb/columnstore/mysql/my.cnf -u root'
 alias ma=/usr/local/mariadb/columnstore/bin/mcsadmin
 alias mcsadmin=/usr/local/mariadb/columnstore/bin/mcsadmin
+alias cpimport=/usr/local/mariadb/columnstore/bin/cpimport
 alias home='cd /usr/local/mariadb/columnstore'
 alias log='cd /var/log/mariadb/columnstore/'
 alias core='cd /var/log/mariadb/columnstore/corefiles'

From f586f3b46a562d717aafa7810087b326e64a2c77 Mon Sep 17 00:00:00 2001
From: Andrew Hutchings <andrew@linuxjedi.co.uk>
Date: Tue, 29 Nov 2016 10:45:38 +0000
Subject: [PATCH 53/54] MCOL-423 Fixes to I_S tables

Fixes the following:

* Compression ratio calculation was incorrect
* Possible issues due to system catalog thread ID usage
* Compressed file size data count was leaking many FDs when the table
  wasn't compressed
* Compressed file size data count was allocating random large amounts
  of RAM and then leaking it when the table wasn't compressed
---
 dbcon/mysql/columnstore_info.sql             |  2 +-
 dbcon/mysql/is_columnstore_columns.cpp       |  2 +-
 dbcon/mysql/is_columnstore_tables.cpp        |  2 +-
 utils/idbdatafile/PosixFileSystem.cpp        | 19 ++++++++++++++++++-
 utils/idbhdfs/hdfs-shared/HdfsFileSystem.cpp | 19 ++++++++++++++++++-
 5 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/dbcon/mysql/columnstore_info.sql b/dbcon/mysql/columnstore_info.sql
index b545ce647..d900c99b0 100644
--- a/dbcon/mysql/columnstore_info.sql
+++ b/dbcon/mysql/columnstore_info.sql
@@ -42,7 +42,7 @@ END //
     
 CREATE PROCEDURE compression_ratio()
 BEGIN
-SELECT CONCAT(((1 - (sum(data_size) / sum(compressed_data_size))) * 100), '%') COMPRESSION_RATIO FROM INFORMATION_SCHEMA.COLUMNSTORE_EXTENTS ce
+SELECT CONCAT(((sum(compressed_data_size) / sum(data_size)) * 100), '%') COMPRESSION_RATIO FROM INFORMATION_SCHEMA.COLUMNSTORE_EXTENTS ce
 JOIN INFORMATION_SCHEMA.COLUMNSTORE_FILES cf ON ce.object_id = cf.object_id
 WHERE compressed_data_size IS NOT NULL;
 END //
diff --git a/dbcon/mysql/is_columnstore_columns.cpp b/dbcon/mysql/is_columnstore_columns.cpp
index d97ca8823..ea9e80aa3 100644
--- a/dbcon/mysql/is_columnstore_columns.cpp
+++ b/dbcon/mysql/is_columnstore_columns.cpp
@@ -60,7 +60,7 @@ static int is_columnstore_columns_fill(THD *thd, TABLE_LIST *tables, COND *cond)
     TABLE *table = tables->table;
 
     boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr =
-        execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(0);
+        execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id));
 
     const std::vector< std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> > catalog_tables
         = systemCatalogPtr->getTables();
diff --git a/dbcon/mysql/is_columnstore_tables.cpp b/dbcon/mysql/is_columnstore_tables.cpp
index e38c8d5e7..5b3bc7fc6 100644
--- a/dbcon/mysql/is_columnstore_tables.cpp
+++ b/dbcon/mysql/is_columnstore_tables.cpp
@@ -48,7 +48,7 @@ static int is_columnstore_tables_fill(THD *thd, TABLE_LIST *tables, COND *cond)
     TABLE *table = tables->table;
 
     boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr =
-        execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(0);
+        execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id));
 
     const std::vector< std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> > catalog_tables
         = systemCatalogPtr->getTables();
diff --git a/utils/idbdatafile/PosixFileSystem.cpp b/utils/idbdatafile/PosixFileSystem.cpp
index 59685c1f9..01d542086 100644
--- a/utils/idbdatafile/PosixFileSystem.cpp
+++ b/utils/idbdatafile/PosixFileSystem.cpp
@@ -159,7 +159,7 @@ size_t readFillBuffer(
 
 off64_t PosixFileSystem::compressedSize(const char *path) const
 {
-    IDBDataFile *pFile = 0;
+    IDBDataFile *pFile = NULL;
     size_t nBytes;
     off64_t dataSize = 0;
 
@@ -178,6 +178,14 @@ off64_t PosixFileSystem::compressedSize(const char *path) const
         nBytes = readFillBuffer( pFile,hdr1,compress::IDBCompressInterface::HDR_BUF_LEN);
         if ( nBytes != compress::IDBCompressInterface::HDR_BUF_LEN )
         {
+            delete pFile;
+            return -1;
+        }
+
+        // Verify we are a compressed file
+        if (decompressor.verifyHdr(hdr1) < 0)
+        {
+            delete pFile;
             return -1;
         }
 
@@ -186,6 +194,8 @@ off64_t PosixFileSystem::compressedSize(const char *path) const
         nBytes = readFillBuffer( pFile,hdr2,ptrSecSize);
         if ( (int64_t)nBytes != ptrSecSize )
         {
+            delete[] hdr2;
+            delete pFile;
             return -1;
         }
 
@@ -194,17 +204,24 @@ off64_t PosixFileSystem::compressedSize(const char *path) const
         delete[] hdr2;
         if (rc != 0)
         {
+            delete pFile;
             return -1;
         }
 
         unsigned k = chunkPtrs.size();
         // last header's offset + length will be the data bytes
+        if (k < 1)
+        {
+            delete pFile;
+            return -1;
+        }
         dataSize = chunkPtrs[k-1].first + chunkPtrs[k-1].second;
         delete pFile;
         return dataSize;
     }
     catch (...)
     {
+        delete pFile;
         return -1;
     }
 }
diff --git a/utils/idbhdfs/hdfs-shared/HdfsFileSystem.cpp b/utils/idbhdfs/hdfs-shared/HdfsFileSystem.cpp
index 1e5626147..a3a874b38 100644
--- a/utils/idbhdfs/hdfs-shared/HdfsFileSystem.cpp
+++ b/utils/idbhdfs/hdfs-shared/HdfsFileSystem.cpp
@@ -151,7 +151,7 @@ size_t readFillBuffer(
 
 off64_t HdfsFileSystem::compressedSize(const char *path) const
 {
-    IDBDataFile *pFile = 0;
+    IDBDataFile *pFile = NULL;
     size_t nBytes;
     off64_t dataSize = 0;
 
@@ -170,6 +170,14 @@ off64_t HdfsFileSystem::compressedSize(const char *path) const
         nBytes = readFillBuffer( pFile,hdr1,compress::IDBCompressInterface::HDR_BUF_LEN);
         if ( nBytes != compress::IDBCompressInterface::HDR_BUF_LEN )
         {
+            delete pFile;
+            return -1;
+        }
+
+        // Verify we are a compressed file
+        if (decompressor.verifyHdr(hdr1) < 0)
+        {
+            delete pFile;
             return -1;
         }
 
@@ -178,6 +186,8 @@ off64_t HdfsFileSystem::compressedSize(const char *path) const
         nBytes = readFillBuffer( pFile,hdr2,ptrSecSize);
         if ( (int64_t)nBytes != ptrSecSize )
         {
+            delete[] hdr2;
+            delete pFile;
             return -1;
         }
 
@@ -186,17 +196,24 @@ off64_t HdfsFileSystem::compressedSize(const char *path) const
         delete[] hdr2;
         if (rc != 0)
         {
+            delete pFile;
             return -1;
         }
 
         unsigned k = chunkPtrs.size();
         // last header's offset + length will be the data bytes
+        if (k < 1)
+        {
+            delete pFile;
+            return -1;
+        }
         dataSize = chunkPtrs[k-1].first + chunkPtrs[k-1].second;
         delete pFile;
         return dataSize;
     }
     catch (...)
     {
+        delete pFile;
         return -1;
     }
 }

From 54b86436fdb13079fcc53c11b562b111a1e6fc01 Mon Sep 17 00:00:00 2001
From: david hill <david.hill@mariadb.com>
Date: Tue, 29 Nov 2016 14:18:57 -0600
Subject: [PATCH 54/54] getSystemNetworkConfig: removed the NIC status and the
 external devices, not needed/used

---
 oamapps/mcsadmin/mcsadmin.cpp | 70 -----------------------------------
 1 file changed, 70 deletions(-)

diff --git a/oamapps/mcsadmin/mcsadmin.cpp b/oamapps/mcsadmin/mcsadmin.cpp
index 50af96f0b..327667203 100644
--- a/oamapps/mcsadmin/mcsadmin.cpp
+++ b/oamapps/mcsadmin/mcsadmin.cpp
@@ -4424,7 +4424,6 @@ int processCommand(string* arguments)
 			cout.width(20);
 			cout << "IP Address";
 			cout.width(14);
-			cout << "Status";
 			if ( AmazonElasticIPCount > 0 )
 			{
 				cout.width(20);
@@ -4503,16 +4502,6 @@ int processCommand(string* arguments)
 								cout << ipAddr;
 								cout.width(14);
 
-								try {
-									oam.getNICStatus(hostname, state);
-		
-									printState(state, " ");
-								}
-								catch (exception& e)
-								{
-									cout << INITIALSTATE;
-								}
-
 								if ( nicID == "1" && AmazonElasticIPCount > 0 )
 								{
 									int id = 1;
@@ -4583,65 +4572,6 @@ int processCommand(string* arguments)
 
 			cout << endl;
 
-			// get and all display Ext Devices Name config parameters
-
-			try
-			{
-	            SystemExtDeviceConfig systemextdeviceconfig;
-				oam.getSystemConfig(systemextdeviceconfig);
-		
-				if ( systemextdeviceconfig.Count == 0 )
-					break;
-
-				cout << endl << "External Device Configuration" << endl << endl;
-
-				cout.setf(ios::left);
-				cout.width(30);
-				cout << "Device Name";
-				cout.width(20);
-				cout << "IP Address";
-				cout.width(10);
-				cout << "Status";
-				cout << endl;
-				cout.width(30);
-				cout << "---------------------";
-				cout.width(20);
-				cout << "---------------";
-				cout.width(12);
-				cout << "------------";
-				cout << endl;
-
-				for ( unsigned int i = 0 ; i < systemextdeviceconfig.Count ; i++ )
-				{
-					cout.setf(ios::left);
-					cout.width(30);
-					cout << systemextdeviceconfig.extdeviceconfig[i].Name;
-					cout.width(20);
-					cout << systemextdeviceconfig.extdeviceconfig[i].IPAddr;
-					cout.width(12);
-
-					int state;
-					try {
-						oam.getExtDeviceStatus(systemextdeviceconfig.extdeviceconfig[i].Name, state);
-
-						printState(state, " ");
-					}
-					catch (exception& e)
-					{
-						cout << INITIALSTATE;
-					}
-					cout << endl;
-				}
-
-				cout << endl;
-			}
-			catch (exception& e)
-			{
-				cout << endl << "**** getextdeviceconfig Failed =  " << e.what() << endl;
-			}
-
-			cout << endl;
-
 			break;
 		}