Merge branch 'develop-1.1' into 1.1-merge-up-20180621

dbcon/joblist/crossenginestep.cpp

@@ -744,7 +744,6 @@ string CrossEngineStep::makeQuery()
 
     // the string must consist of a single SQL statement without a terminating semicolon ; or \g.
     // oss << ";";
-
     return oss.str();
 }
 

dbcon/mysql/columnstore_info.sql

@@ -37,43 +37,56 @@ DROP PROCEDURE IF EXISTS `table_usage` //
 CREATE PROCEDURE table_usage (IN t_schema char(64), IN t_name char(64))
 `table_usage`: BEGIN
 
+    DECLARE done INTEGER DEFAULT 0;
+    DECLARE dbname VARCHAR(64);
+    DECLARE tbname VARCHAR(64);
+    DECLARE object_ids TEXT;
+    DECLARE dictionary_object_ids TEXT;
     DECLARE `locker` TINYINT UNSIGNED DEFAULT IS_USED_LOCK('table_usage');
-
+    DECLARE columns_list CURSOR FOR SELECT TABLE_SCHEMA, TABLE_NAME, GROUP_CONCAT(object_id) OBJECT_IDS, GROUP_CONCAT(dictionary_object_id) DICT_OBJECT_IDS FROM INFORMATION_SCHEMA.COLUMNSTORE_COLUMNS WHERE table_name = t_name and table_schema = t_schema  GROUP BY table_schema, table_name;
+    DECLARE columns_list_sc CURSOR FOR SELECT TABLE_SCHEMA, TABLE_NAME, GROUP_CONCAT(object_id) OBJECT_IDS, GROUP_CONCAT(dictionary_object_id) DICT_OBJECT_IDS FROM INFORMATION_SCHEMA.COLUMNSTORE_COLUMNS WHERE table_schema = t_schema  GROUP BY table_schema, table_name;
+    DECLARE columns_list_all CURSOR FOR SELECT TABLE_SCHEMA, TABLE_NAME, GROUP_CONCAT(object_id) OBJECT_IDS, GROUP_CONCAT(dictionary_object_id) DICT_OBJECT_IDS FROM INFORMATION_SCHEMA.COLUMNSTORE_COLUMNS  GROUP BY table_schema, table_name;
+    DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = 1;
     IF `locker` IS NOT NULL THEN
         SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Error acquiring table_usage lock';
         LEAVE `table_usage`;
     END IF;
     DO GET_LOCK('table_usage', 0);
-    DROP TABLE IF EXISTS columnstore_info.columnstore_columns;
     DROP TABLE IF EXISTS columnstore_info.columnstore_files;
-    CREATE TABLE columnstore_info.columnstore_columns engine=myisam as (select * from information_schema.columnstore_columns);
+    CREATE TEMPORARY TABLE columnstore_info.columnstore_files (TABLE_SCHEMA VARCHAR(64), TABLE_NAME VARCHAR(64), DATA BIGINT, DICT BIGINT);
-    ALTER TABLE columnstore_info.columnstore_columns ADD INDEX `object_id` (`object_id`);
+
-    ALTER TABLE columnstore_info.columnstore_columns ADD INDEX `dictionary_object_id` (`dictionary_object_id`);
-    CREATE TABLE columnstore_info.columnstore_files engine=myisam as (select * from information_schema.columnstore_files);
-    ALTER TABLE columnstore_info.columnstore_files ADD INDEX `object_id` (`object_id`);
     IF t_name IS NOT NULL THEN
-SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + COALESCE(dict, 0)) as TOTAL_USAGE FROM (
+        OPEN columns_list;
-SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict
-FROM
-columnstore_info.columnstore_columns ics where table_name = t_name and (table_schema = t_schema or t_schema IS NULL)
-group by table_schema, table_name
-) q;
     ELSEIF t_schema IS NOT NULL THEN
-SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + COALESCE(dict, 0)) as TOTAL_USAGE FROM (
+        OPEN columns_list_sc;
-SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict
-FROM
-columnstore_info.columnstore_columns ics where table_schema = t_schema
-group by table_schema, table_name
-) q;
     ELSE
-SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(data) as DATA_DISK_USAGE, columnstore_info.format_filesize(dict) as DICT_DISK_USAGE, columnstore_info.format_filesize(data + COALESCE(dict, 0)) as TOTAL_USAGE FROM (
+        OPEN columns_list_all;
-SELECT TABLE_SCHEMA, TABLE_NAME, (SELECT sum(cf.file_size) as data FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema) as data, (SELECT sum(cf.file_size) as dict FROM columnstore_info.columnstore_columns cc JOIN columnstore_info.columnstore_files cf ON cc.dictionary_object_id = cf.object_id WHERE table_name = ics.table_name and table_schema = ics.table_schema GROUP BY table_schema, table_name) as dict
-FROM
-columnstore_info.columnstore_columns ics
-group by table_schema, table_name
-) q;
     END IF;
-    DROP TABLE IF EXISTS columnstore_info.columnstore_columns;
+
+    files_table: LOOP
+	IF t_name IS NOT NULL THEN
+            FETCH columns_list INTO dbname, tbname, object_ids, dictionary_object_ids;
+        ELSEIF t_schema IS NOT NULL THEN
+            FETCH columns_list_sc INTO dbname, tbname, object_ids, dictionary_object_ids;
+        ELSE
+            FETCH columns_list_all INTO dbname, tbname, object_ids, dictionary_object_ids;
+        END IF;
+        IF done = 1 THEN LEAVE files_table;
+        END IF;
+        INSERT INTO columnstore_info.columnstore_files (SELECT dbname, tbname, sum(file_size), 0 FROM information_schema.columnstore_files WHERE find_in_set(object_id, object_ids));
+        IF dictionary_object_ids IS NOT NULL THEN
+    	UPDATE columnstore_info.columnstore_files SET DICT = (SELECT sum(file_size) FROM information_schema.columnstore_files WHERE find_in_set(object_id, dictionary_object_ids)) WHERE TABLE_SCHEMA = dbname AND TABLE_NAME = tbname;
+        END IF;
+    END LOOP;
+    IF t_name IS NOT NULL THEN
+        CLOSE columns_list;
+    ELSEIF t_schema IS NOT NULL THEN
+        CLOSE columns_list_sc;
+    ELSE
+        CLOSE columns_list_all;
+    END IF;
+    SELECT TABLE_SCHEMA, TABLE_NAME, columnstore_info.format_filesize(DATA) as DATA_DISK_USAGE, columnstore_info.format_filesize(DICT) as DICT_DATA_USAGE, columnstore_info.format_filesize(DATA + COALESCE(DICT, 0)) as TOTAL_USAGE FROM columnstore_info.columnstore_files;
+
     DROP TABLE IF EXISTS columnstore_info.columnstore_files;
     DO RELEASE_LOCK('table_usage');
 END //

dbcon/mysql/is_columnstore_columns.cpp

@@ -56,10 +56,59 @@ ST_FIELD_INFO is_columnstore_columns_fields[] =
 
 };
 
+static void get_cond_item(Item_func *item, String **table, String **db)
+{
+    char tmp_char[MAX_FIELD_WIDTH];
+    Item_field *item_field = (Item_field*) item->arguments()[0]->real_item();
+    if (strcasecmp(item_field->field_name, "table_name") == 0)
+    {
+        String str_buf(tmp_char, sizeof(tmp_char), system_charset_info);
+        *table = item->arguments()[1]->val_str(&str_buf);
+        return;
+    }
+    else if (strcasecmp(item_field->field_name, "table_schema") == 0)
+    {
+        String str_buf(tmp_char, sizeof(tmp_char), system_charset_info);
+        *db = item->arguments()[1]->val_str(&str_buf);
+        return;
+    }
+}
+
+static void get_cond_items(COND *cond, String **table, String **db)
+{
+    if (cond->type() == Item::FUNC_ITEM)
+    {
+        Item_func* fitem = (Item_func*) cond;
+        if (fitem->arguments()[0]->real_item()->type() == Item::FIELD_ITEM &&
+            fitem->arguments()[1]->const_item())
+        {
+            get_cond_item(fitem, table, db);
+        }
+    }
+    else if ((cond->type() == Item::COND_ITEM) && (((Item_cond*) cond)->functype() == Item_func::COND_AND_FUNC))
+    {
+        List_iterator<Item> li(*((Item_cond*) cond)->argument_list());
+        Item *item;
+        while ((item= li++))
+        {
+            if (item->type() == Item::FUNC_ITEM)
+            {
+                get_cond_item((Item_func*)item, table, db);
+            }
+            else
+            {
+                get_cond_items(item, table, db);
+            }
+        }
+    }
+}
+
 static int is_columnstore_columns_fill(THD* thd, TABLE_LIST* tables, COND* cond)
 {
     CHARSET_INFO* cs = system_charset_info;
     TABLE* table = tables->table;
+    String *table_name = NULL;
+    String *db_name = NULL;
 
     boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr =
         execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id));
@@ -69,9 +118,29 @@ static int is_columnstore_columns_fill(THD* thd, TABLE_LIST* tables, COND* cond)
 
     systemCatalogPtr->identity(execplan::CalpontSystemCatalog::FE);
 
+    if (cond)
+    {
+        get_cond_items(cond, &table_name, &db_name);
+    }
+
     for (std::vector<std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> >::const_iterator it = catalog_tables.begin();
             it != catalog_tables.end(); ++it)
     {
+        if (db_name)
+        {
+            if ((*it).second.schema.compare(db_name->ptr()) != 0)
+            {
+                continue;
+            }
+        }
+        if (table_name)
+        {
+            if ((*it).second.table.compare(table_name->ptr()) != 0)
+            {
+                continue;
+            }
+        }
+
         execplan::CalpontSystemCatalog::RIDList column_rid_list;
 
         // Note a table may get dropped as you iterate over the list of tables.
@@ -184,8 +253,6 @@ static int is_columnstore_columns_fill(THD* thd, TABLE_LIST* tables, COND* cond)
         }
     }
 
-
-
     return 0;
 }
 

dbcon/mysql/is_columnstore_extents.cpp

@@ -52,30 +52,18 @@ ST_FIELD_INFO is_columnstore_extents_fields[] =
     {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0}
 };
 
-static int is_columnstore_extents_fill(THD* thd, TABLE_LIST* tables, COND* cond)
+static int generate_result(BRM::OID_t oid, BRM::DBRM *emp, TABLE *table, THD *thd)
 {
     CHARSET_INFO* cs = system_charset_info;
-    TABLE* table = tables->table;
     std::vector<struct BRM::EMEntry> entries;
     std::vector<struct BRM::EMEntry>::iterator iter;
     std::vector<struct BRM::EMEntry>::iterator end;
 
-    BRM::DBRM* emp = new BRM::DBRM();
 
-    if (!emp || !emp->isDBRMReady())
-    {
-        return 1;
-    }
-
-    execplan::ObjectIDManager oidm;
-    BRM::OID_t MaxOID = oidm.size();
-
-    for (BRM::OID_t oid = 3000; oid <= MaxOID; oid++)
-    {
     emp->getExtents(oid, entries, false, false, true);
 
     if (entries.size() == 0)
-            continue;
+        return 0;
 
     iter = entries.begin();
     end = entries.end();
@@ -191,8 +179,89 @@ static int is_columnstore_extents_fill(THD* thd, TABLE_LIST* tables, COND* cond)
         iter++;
 
     }
+    return 0;
+}
+
+static int is_columnstore_extents_fill(THD *thd, TABLE_LIST *tables, COND *cond)
+{
+    BRM::OID_t cond_oid = 0;
+    TABLE *table = tables->table;
+
+    BRM::DBRM *emp = new BRM::DBRM();
+    if (!emp || !emp->isDBRMReady())
+    {
+        return 1;
     }
 
+    if (cond && cond->type() == Item::FUNC_ITEM)
+    {
+        Item_func* fitem = (Item_func*) cond;
+        if ((fitem->functype() == Item_func::EQ_FUNC) && (fitem->argument_count() == 2))
+        {
+            if(fitem->arguments()[0]->real_item()->type() == Item::FIELD_ITEM &&
+               fitem->arguments()[1]->const_item())
+            {
+                // WHERE object_id = value
+                Item_field *item_field = (Item_field*) fitem->arguments()[0]->real_item();
+                if (strcasecmp(item_field->field_name, "object_id") == 0)
+                {
+                    cond_oid = fitem->arguments()[1]->val_int();
+                    return generate_result(cond_oid, emp, table, thd);
+                }
+            }
+            else if (fitem->arguments()[1]->real_item()->type() == Item::FIELD_ITEM &&
+                fitem->arguments()[0]->const_item())
+            {
+                // WHERE value = object_id
+                Item_field *item_field = (Item_field*) fitem->arguments()[1]->real_item();
+                if (strcasecmp(item_field->field_name, "object_id") == 0)
+                {
+                    cond_oid = fitem->arguments()[0]->val_int();
+                    return generate_result(cond_oid, emp, table, thd);
+                }
+            }
+        }
+        else if (fitem->functype() == Item_func::IN_FUNC)
+        {
+            // WHERE object_id in (value1, value2)
+            Item_field *item_field = (Item_field*) fitem->arguments()[0]->real_item();
+            if (strcasecmp(item_field->field_name, "object_id") == 0)
+            {
+                for (unsigned int i=1; i < fitem->argument_count(); i++)
+                {
+                    cond_oid = fitem->arguments()[i]->val_int();
+                    int result = generate_result(cond_oid, emp, table, thd);
+                    if (result)
+                       return 1;
+                }
+            }
+        }
+        else if (fitem->functype() == Item_func::UNKNOWN_FUNC &&
+                 strcasecmp(fitem->func_name(), "find_in_set") == 0)
+        {
+            // WHERE FIND_IN_SET(object_id, values)
+            String *tmp_var = fitem->arguments()[1]->val_str();
+            std::stringstream ss(tmp_var->ptr());
+            while (ss >> cond_oid)
+            {
+                int ret = generate_result(cond_oid, emp, table, thd);
+                if (ret)
+                    return 1;
+                if (ss.peek() == ',')
+                    ss.ignore();
+            }
+        }
+    }
+
+    execplan::ObjectIDManager oidm;
+    BRM::OID_t MaxOID = oidm.size();
+
+    for(BRM::OID_t oid = 3000; oid <= MaxOID; oid++)
+    {
+        int result = generate_result(oid, emp, table, thd);
+        if (result)
+            return 1;
+    }
     delete emp;
     return 0;
 }

dbcon/mysql/is_columnstore_files.cpp

@@ -84,12 +84,10 @@ static bool get_file_sizes(messageqcpp::MessageQueueClient* msgQueueClient, cons
     }
 }
 
-static int is_columnstore_files_fill(THD* thd, TABLE_LIST* tables, COND* cond)
+static int generate_result(BRM::OID_t oid, BRM::DBRM *emp, TABLE *table, THD *thd)
 {
-    BRM::DBRM* emp = new BRM::DBRM();
     std::vector<struct BRM::EMEntry> entries;
     CHARSET_INFO* cs = system_charset_info;
-    TABLE* table = tables->table;
 
     char oidDirName[WriteEngine::FILE_NAME_SIZE];
     char fullFileName[WriteEngine::FILE_NAME_SIZE];
@@ -103,20 +101,10 @@ static int is_columnstore_files_fill(THD* thd, TABLE_LIST* tables, COND* cond)
     oam::Oam oam_instance;
     int pmId = 0;
 
-    if (!emp || !emp->isDBRMReady())
-    {
-        return 1;
-    }
-
-    execplan::ObjectIDManager oidm;
-    BRM::OID_t MaxOID = oidm.size();
-
-    for (BRM::OID_t oid = 3000; oid <= MaxOID; oid++)
-    {
     emp->getExtents(oid, entries, false, false, true);
 
     if (entries.size() == 0)
-            continue;
+        return 0;
 
     std::vector<struct BRM::EMEntry>::const_iterator iter = entries.begin();
 
@@ -126,7 +114,7 @@ static int is_columnstore_files_fill(THD* thd, TABLE_LIST* tables, COND* cond)
         if (iter->blockOffset > 0)
         {
             iter++;
-                continue;
+            return 0;
         }
 
         try
@@ -137,7 +125,7 @@ static int is_columnstore_files_fill(THD* thd, TABLE_LIST* tables, COND* cond)
         {
             // MCOL-1116: If we are here a DBRoot is offline/missing
             iter++;
-                continue;
+            return 0;
         }
 
         table->field[0]->store(oid);
@@ -197,6 +185,91 @@ static int is_columnstore_files_fill(THD* thd, TABLE_LIST* tables, COND* cond)
         messageqcpp::MessageQueueClientPool::releaseInstance(msgQueueClient);
         msgQueueClient = NULL;
     }
+    return 0;
+}
+
+static int is_columnstore_files_fill(THD *thd, TABLE_LIST *tables, COND *cond)
+{
+    BRM::DBRM *emp = new BRM::DBRM();
+    BRM::OID_t cond_oid = 0;
+    TABLE *table = tables->table;
+
+    if (!emp || !emp->isDBRMReady())
+    {
+        return 1;
+    }
+
+    if (cond && cond->type() == Item::FUNC_ITEM)
+    {
+        Item_func* fitem = (Item_func*) cond;
+        if ((fitem->functype() == Item_func::EQ_FUNC) && (fitem->argument_count() == 2))
+        {
+            if(fitem->arguments()[0]->real_item()->type() == Item::FIELD_ITEM &&
+               fitem->arguments()[1]->const_item())
+            {
+                // WHERE object_id = value
+                Item_field *item_field = (Item_field*) fitem->arguments()[0]->real_item();
+                if (strcasecmp(item_field->field_name, "object_id") == 0)
+                {
+                    cond_oid = fitem->arguments()[1]->val_int();
+                    return generate_result(cond_oid, emp, table, thd);
+                }
+            }
+            else if (fitem->arguments()[1]->real_item()->type() == Item::FIELD_ITEM &&
+                fitem->arguments()[0]->const_item())
+            {
+                // WHERE value = object_id
+                Item_field *item_field = (Item_field*) fitem->arguments()[1]->real_item();
+                if (strcasecmp(item_field->field_name, "object_id") == 0)
+                {
+                    cond_oid = fitem->arguments()[0]->val_int();
+                    return generate_result(cond_oid, emp, table, thd);
+                }
+            }
+        }
+        else if (fitem->functype() == Item_func::IN_FUNC)
+        {
+            // WHERE object_id in (value1, value2)
+            Item_field *item_field = (Item_field*) fitem->arguments()[0]->real_item();
+            if (strcasecmp(item_field->field_name, "object_id") == 0)
+            {
+                for (unsigned int i=1; i < fitem->argument_count(); i++)
+                {
+                    cond_oid = fitem->arguments()[i]->val_int();
+                    int result = generate_result(cond_oid, emp, table, thd);
+                    if (result)
+                       return 1;
+                }
+            }
+        }
+        else if (fitem->functype() == Item_func::UNKNOWN_FUNC &&
+                 strcasecmp(fitem->func_name(), "find_in_set") == 0)
+        {
+            // WHERE FIND_IN_SET(object_id, values)
+            String *tmp_var = fitem->arguments()[1]->val_str();
+            std::stringstream ss(tmp_var->ptr());
+            while (ss >> cond_oid)
+            {
+                int ret = generate_result(cond_oid, emp, table, thd);
+                if (ret)
+                    return 1;
+                if (ss.peek() == ',')
+                    ss.ignore();
+            }
+        }
+    }
+
+    execplan::ObjectIDManager oidm;
+    BRM::OID_t MaxOID = oidm.size();
+
+    if (!cond_oid)
+    {
+        for(BRM::OID_t oid = 3000; oid <= MaxOID; oid++)
+        {
+            int result = generate_result(oid, emp, table, thd);
+            if (result)
+                return 1;
+        }
     }
 
     delete emp;

dbcon/mysql/is_columnstore_tables.cpp

@@ -42,22 +42,91 @@ ST_FIELD_INFO is_columnstore_tables_fields[] =
     {0, 0, MYSQL_TYPE_NULL, 0, 0, 0, 0}
 };
 
+static void get_cond_item(Item_func *item, String **table, String **db)
+{
+    char tmp_char[MAX_FIELD_WIDTH];
+    Item_field *item_field = (Item_field*) item->arguments()[0]->real_item();
+    if (strcasecmp(item_field->field_name, "table_name") == 0)
+    {
+        String str_buf(tmp_char, sizeof(tmp_char), system_charset_info);
+        *table = item->arguments()[1]->val_str(&str_buf);
+        return;
+    }
+    else if (strcasecmp(item_field->field_name, "table_schema") == 0)
+    {
+        String str_buf(tmp_char, sizeof(tmp_char), system_charset_info);
+        *db = item->arguments()[1]->val_str(&str_buf);
+        return;
+    }
+}
+
+static void get_cond_items(COND *cond, String **table, String **db)
+{
+    if (cond->type() == Item::FUNC_ITEM)
+    {
+        Item_func* fitem = (Item_func*) cond;
+        if (fitem->arguments()[0]->real_item()->type() == Item::FIELD_ITEM &&
+            fitem->arguments()[1]->const_item())
+        {
+            get_cond_item(fitem, table, db);
+        }
+    }
+    else if ((cond->type() == Item::COND_ITEM) && (((Item_cond*) cond)->functype() == Item_func::COND_AND_FUNC))
+    {
+        List_iterator<Item> li(*((Item_cond*) cond)->argument_list());
+        Item *item;
+        while ((item= li++))
+        {
+            if (item->type() == Item::FUNC_ITEM)
+            {
+                get_cond_item((Item_func*)item, table, db);
+            }
+            else
+            {
+                get_cond_items(item, table, db);
+            }
+        }
+    }
+}
+
 static int is_columnstore_tables_fill(THD* thd, TABLE_LIST* tables, COND* cond)
 {
     CHARSET_INFO* cs = system_charset_info;
     TABLE* table = tables->table;
+    String *table_name = NULL;
+    String *db_name = NULL;
 
     boost::shared_ptr<execplan::CalpontSystemCatalog> systemCatalogPtr =
         execplan::CalpontSystemCatalog::makeCalpontSystemCatalog(execplan::CalpontSystemCatalog::idb_tid2sid(thd->thread_id));
 
     systemCatalogPtr->identity(execplan::CalpontSystemCatalog::FE);
 
+    if (cond)
+    {
+        get_cond_items(cond, &table_name, &db_name);
+    }
+
     const std::vector< std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> > catalog_tables
         = systemCatalogPtr->getTables();
 
     for (std::vector<std::pair<execplan::CalpontSystemCatalog::OID, execplan::CalpontSystemCatalog::TableName> >::const_iterator it = catalog_tables.begin();
             it != catalog_tables.end(); ++it)
     {
+        if (db_name)
+        {
+            if ((*it).second.schema.compare(db_name->ptr()) != 0)
+            {
+                continue;
+            }
+        }
+        if (table_name)
+        {
+            if ((*it).second.table.compare(table_name->ptr()) != 0)
+            {
+                continue;
+            }
+        }
+
         execplan::CalpontSystemCatalog::TableInfo tb_info = systemCatalogPtr->tableInfo((*it).second);
         std::string create_date = dataconvert::DataConvert::dateToString((*it).second.create_date);
         table->field[0]->store((*it).second.schema.c_str(), (*it).second.schema.length(), cs);

oam/cloud/MCSVolumeCmds.sh

@@ -202,7 +202,7 @@ detachvolume() {
 	checkInfostatus
 	if [ $STATUS == "detaching" ]; then
 		retries=1
-		while [ $retries -ne 60 ]; do
+		while [ $retries -ne 10 ]; do
 			#retry until it's attached
 			$AWSCLI detach-volume --volume-id  $volumeName --region $Region > /tmp/volumeInfo_$volumeName 2>&1
 		
@@ -239,7 +239,7 @@ attachvolume() {
 	checkInfostatus
 	if [ $STATUS == "attaching" -o $STATUS == "already-attached" ]; then
 		retries=1
-		while [ $retries -ne 60 ]; do
+		while [ $retries -ne 10 ]; do
 			#check status until it's attached
 			describevolume
 			if [ $STATUS == "attached" ]; then

oam/oamcpp/liboamcpp.cpp

@@ -5811,6 +5811,35 @@ bool Oam::autoMovePmDbroot(std::string residePM)
         exceptionControl("autoMovePmDbroot", API_INVALID_PARAMETER);
     }
 
+		//detach first to make sure DBS can be detach before trying to move to another pm
+		DBRootConfigList::iterator pt3 = residedbrootConfigList.begin();
+		for( ; pt3 != residedbrootConfigList.end() ; pt3++ )
+		{
+			int dbrootID = *pt3;
+
+			try
+			{
+				typedef std::vector<string> dbrootList;
+				dbrootList dbrootlist;
+				dbrootlist.push_back(itoa(dbrootID));
+
+				amazonDetach(dbrootlist);
+			}
+			catch (exception& )
+			{
+				writeLog("ERROR: amazonDetach failure", LOG_TYPE_ERROR );
+				
+				//reattach
+				typedef std::vector<string> dbrootList;
+				dbrootList dbrootlist;
+				dbrootlist.push_back(itoa(dbrootID));
+
+				amazonAttach(residePM, dbrootlist);
+
+				exceptionControl("autoMovePmDbroot", API_DETACH_FAILURE);
+			}
+		}
+
     //get dbroot id for other PMs
     systemStorageInfo_t t;
     DeviceDBRootList moduledbrootlist;
@@ -6330,10 +6359,8 @@ bool Oam::autoUnMovePmDbroot(std::string toPM)
     }
 
     if (!found)
-    {
+			writeLog("No dbroots found in ../Calpont/local/moveDbrootTransactionLog", LOG_TYPE_DEBUG );
-        writeLog("ERROR: no dbroots found in ../Calpont/local/moveDbrootTransactionLog", LOG_TYPE_ERROR );
+			cout << "No dbroots found in " << fileName << endl;
-        cout << "ERROR: no dbroots found in " << fileName << endl;
-        exceptionControl("autoUnMovePmDbroot", API_FAILURE);
     }
 
     oldFile.close();
@@ -7746,7 +7773,7 @@ void Oam::actionMysqlCalpont(MYSQLCALPONT_ACTION action)
     else
         return;
 
-    // check if mysql-Capont is installed
+		// check if mysql-Columnstore is installed
     string mysqlscript = InstallDir + "/mysql/mysql-Columnstore";
 
     if (access(mysqlscript.c_str(), X_OK) != 0)
@@ -10343,6 +10370,146 @@ void Oam::sendStatusUpdate(ByteStream obs, ByteStream::byte returnRequestType)
 }
 
 /***************************************************************************
+ *
+     * Function:  amazonDetach
+     *
+     * Purpose:   Amazon EC2 volume deattach needed
+     *
+     ****************************************************************************/
+
+    void Oam::amazonDetach(dbrootList dbrootConfigList)
+    {
+		//if amazon cloud with external volumes, do the detach/attach moves
+		string cloud;
+		string DBRootStorageType;
+		try {
+			getSystemConfig("Cloud", cloud);
+			getSystemConfig("DBRootStorageType", DBRootStorageType);
+		}
+		catch(...) {}
+
+		if ( (cloud == "amazon-ec2" || cloud == "amazon-vpc") && 
+			DBRootStorageType == "external" )
+		{
+			writeLog("amazonDetach function started ", LOG_TYPE_DEBUG );
+
+			dbrootList::iterator pt3 = dbrootConfigList.begin();
+			for( ; pt3 != dbrootConfigList.end() ; pt3++)
+			{
+				string dbrootid = *pt3;
+				string volumeNameID = "PMVolumeName" + dbrootid;
+				string volumeName = oam::UnassignedName;
+				string deviceNameID = "PMVolumeDeviceName" + dbrootid;
+				string deviceName = oam::UnassignedName;
+				try {
+					getSystemConfig( volumeNameID, volumeName);
+					getSystemConfig( deviceNameID, deviceName);
+				}
+				catch(...)
+				{}
+
+				if ( volumeName == oam::UnassignedName || deviceName == oam::UnassignedName )
+				{
+					cout << "   ERROR: amazonDetach, invalid configure " + volumeName + ":" + deviceName << endl;
+					writeLog("ERROR: amazonDetach, invalid configure " + volumeName + ":" + deviceName, LOG_TYPE_ERROR );
+					exceptionControl("amazonDetach", API_INVALID_PARAMETER);
+				}
+
+				//send msg to to-pm to umount volume
+				int returnStatus = sendMsgToProcMgr(UNMOUNT, dbrootid, FORCEFUL, ACK_YES);
+				if (returnStatus != API_SUCCESS) {
+					writeLog("ERROR: amazonDetach, umount failed on " + dbrootid, LOG_TYPE_ERROR );
+				}
+
+				if (!detachEC2Volume(volumeName)) {
+					cout << "   ERROR: amazonDetach, detachEC2Volume failed on " + volumeName << endl;
+					writeLog("ERROR: amazonDetach, detachEC2Volume failed on " + volumeName , LOG_TYPE_ERROR );
+					exceptionControl("amazonDetach", API_FAILURE);
+				}
+
+				writeLog("amazonDetach, detachEC2Volume passed on " + volumeName , LOG_TYPE_DEBUG );
+			}
+		}
+	}
+
+    /***************************************************************************
+     *
+     * Function:  amazonAttach
+     *
+     * Purpose:   Amazon EC2 volume Attach needed
+     *
+     ****************************************************************************/
+
+    void Oam::amazonAttach(std::string toPM, dbrootList dbrootConfigList)
+    {
+		//if amazon cloud with external volumes, do the detach/attach moves
+		string cloud;
+		string DBRootStorageType;
+		try {
+			getSystemConfig("Cloud", cloud);
+			getSystemConfig("DBRootStorageType", DBRootStorageType);
+		}
+		catch(...) {}
+
+		if ( (cloud == "amazon-ec2" || cloud == "amazon-vpc") && 
+			DBRootStorageType == "external" )
+		{
+			writeLog("amazonAttach function started ", LOG_TYPE_DEBUG );
+
+			//get Instance Name for to-pm
+			string toInstanceName = oam::UnassignedName;
+			try
+			{
+				ModuleConfig moduleconfig;
+				getSystemConfig(toPM, moduleconfig);
+				HostConfigList::iterator pt1 = moduleconfig.hostConfigList.begin();
+				toInstanceName = (*pt1).HostName;
+			}
+			catch(...)
+			{}
+
+			if ( toInstanceName == oam::UnassignedName || toInstanceName.empty() )
+			{
+				cout << "   ERROR: amazonAttach, invalid Instance Name for " << toPM << endl;
+				writeLog("ERROR: amazonAttach, invalid Instance Name " + toPM, LOG_TYPE_ERROR );
+				exceptionControl("amazonAttach", API_INVALID_PARAMETER);
+			}
+
+			dbrootList::iterator pt3 = dbrootConfigList.begin();
+			for( ; pt3 != dbrootConfigList.end() ; pt3++)
+			{
+				string dbrootid = *pt3;
+				string volumeNameID = "PMVolumeName" + dbrootid;
+				string volumeName = oam::UnassignedName;
+				string deviceNameID = "PMVolumeDeviceName" + dbrootid;
+				string deviceName = oam::UnassignedName;
+				try {
+					getSystemConfig( volumeNameID, volumeName);
+					getSystemConfig( deviceNameID, deviceName);
+				}
+				catch(...)
+				{}
+
+				if ( volumeName == oam::UnassignedName || deviceName == oam::UnassignedName )
+				{
+					cout << "   ERROR: amazonAttach, invalid configure " + volumeName + ":" + deviceName << endl;
+					writeLog("ERROR: amazonAttach, invalid configure " + volumeName + ":" + deviceName, LOG_TYPE_ERROR );
+					exceptionControl("amazonAttach", API_INVALID_PARAMETER);
+				}
+
+				if (!attachEC2Volume(volumeName, deviceName, toInstanceName)) {
+					cout << "   ERROR: amazonAttach, attachEC2Volume failed on " + volumeName + ":" + deviceName + ":" + toInstanceName << endl;
+					writeLog("ERROR: amazonAttach, attachEC2Volume failed on " + volumeName + ":" + deviceName + ":" + toInstanceName, LOG_TYPE_ERROR );
+					exceptionControl("amazonAttach", API_FAILURE);
+				}
+
+				writeLog("amazonAttach, attachEC2Volume passed on " + volumeName + ":" + toPM, LOG_TYPE_DEBUG );
+			}
+		}
+	}
+
+
+     /***************************************************************************
      *
  * Function:  amazonReattach
  *
@@ -10445,6 +10612,7 @@ void Oam::amazonReattach(std::string toPM, dbrootList dbrootConfigList, bool att
     }
 }
 
+
 /***************************************************************************
  *
  * Function:  mountDBRoot

oam/oamcpp/liboamcpp.h

@@ -229,6 +229,7 @@ enum API_STATUS
     API_CONN_REFUSED,
     API_CANCELLED,
     API_STILL_WORKING,
+	API_DETACH_FAILURE,
     API_MAX
 };
 
@@ -2432,6 +2433,8 @@ public:
 
     void amazonReattach(std::string toPM, dbrootList dbrootConfigList, bool attach = false);
     void mountDBRoot(dbrootList dbrootConfigList, bool mount = true);
+		void amazonDetach(dbrootList dbrootConfigList);
+		void amazonAttach(std::string toPM, dbrootList dbrootConfigList);
 
     /**
     *@brief  gluster control

primitives/primproc/primitiveserver.cpp

@@ -2067,6 +2067,11 @@ struct ReadThread
                         case DICT_CREATE_EQUALITY_FILTER:
                         {
                             PriorityThreadPool::Job job;
+                        const uint8_t *buf = bs->buf();
+                        uint32_t pos = sizeof(ISMPacketHeader) - 2;
+                        job.stepID = *((uint32_t *) &buf[pos+6]);
+                        job.uniqueID = *((uint32_t *) &buf[pos+10]);
+                        job.sock = outIos;
                             job.functor = boost::shared_ptr<PriorityThreadPool::Functor>(new CreateEqualityFilter(bs));
                             OOBPool->addJob(job);
                             break;
@@ -2075,6 +2080,11 @@ struct ReadThread
                         case DICT_DESTROY_EQUALITY_FILTER:
                         {
                             PriorityThreadPool::Job job;
+                        const uint8_t *buf = bs->buf();
+                        uint32_t pos = sizeof(ISMPacketHeader) - 2;
+                        job.stepID = *((uint32_t *) &buf[pos+6]);
+                        job.uniqueID = *((uint32_t *) &buf[pos+10]);
+                        job.sock = outIos;
                             job.functor = boost::shared_ptr<PriorityThreadPool::Functor>(new DestroyEqualityFilter(bs));
                             OOBPool->addJob(job);
                             break;
@@ -2108,9 +2118,11 @@ struct ReadThread
                             job.id = hdr->Hdr.UniqueID;
                             job.weight = LOGICAL_BLOCK_RIDS;
                             job.priority = hdr->Hdr.Priority;
-
+                        const uint8_t *buf = bs->buf();
-                            if (hdr->flags & IS_SYSCAT)
+                        uint32_t pos = sizeof(ISMPacketHeader) - 2;
-                            {
+                        job.stepID = *((uint32_t *) &buf[pos+6]);
+                        job.uniqueID = *((uint32_t *) &buf[pos+10]);
+                        job.sock = outIos;
                                 //boost::thread t(DictScanJob(outIos, bs, writeLock));
                                 // using already-existing threads may cut latency
                                 // if it's changed back to running in an independent thread
@@ -2155,9 +2167,12 @@ struct ReadThread
                             job.id = bpps->getID();
                             job.weight = ismHdr->Size;
                             job.priority = bpps->priority();
+                        const uint8_t *buf = bs->buf();
+                        uint32_t pos = sizeof(ISMPacketHeader) - 2;
+                        job.stepID = *((uint32_t *) &buf[pos+6]);
+                        job.uniqueID = *((uint32_t *) &buf[pos+10]);
+                        job.sock = outIos;
 
-                            if (bpps->isSysCat())
-                            {
                                 //boost::thread t(*bpps);
                                 // using already-existing threads may cut latency
                                 // if it's changed back to running in an independent thread
@@ -2176,6 +2191,11 @@ struct ReadThread
                         {
                             PriorityThreadPool::Job job;
                             job.functor = boost::shared_ptr<PriorityThreadPool::Functor>(new BPPHandler::Create(fBPPHandler, bs));
+                        const uint8_t *buf = bs->buf();
+                        uint32_t pos = sizeof(ISMPacketHeader) - 2;
+                        job.stepID = *((uint32_t *) &buf[pos+6]);
+                        job.uniqueID = *((uint32_t *) &buf[pos+10]);
+                        job.sock = outIos;
                             OOBPool->addJob(job);
                             //fBPPHandler->createBPP(*bs);
                             break;
@@ -2186,6 +2206,11 @@ struct ReadThread
                             PriorityThreadPool::Job job;
                             job.functor = boost::shared_ptr<PriorityThreadPool::Functor>(new BPPHandler::AddJoiner(fBPPHandler, bs));
                             job.id = fBPPHandler->getUniqueID(bs, ismHdr->Command);
+                        const uint8_t *buf = bs->buf();
+                        uint32_t pos = sizeof(ISMPacketHeader) - 2;
+                        job.stepID = *((uint32_t *) &buf[pos+6]);
+                        job.uniqueID = *((uint32_t *) &buf[pos+10]);
+                        job.sock = outIos;
                             OOBPool->addJob(job);
                             //fBPPHandler->addJoinerToBPP(*bs);
                             break;
@@ -2199,6 +2224,11 @@ struct ReadThread
                             PriorityThreadPool::Job job;
                             job.functor = boost::shared_ptr<PriorityThreadPool::Functor>(new BPPHandler::LastJoiner(fBPPHandler, bs));
                             job.id = fBPPHandler->getUniqueID(bs, ismHdr->Command);
+                        const uint8_t *buf = bs->buf();
+                        uint32_t pos = sizeof(ISMPacketHeader) - 2;
+                        job.stepID = *((uint32_t *) &buf[pos+6]);
+                        job.uniqueID = *((uint32_t *) &buf[pos+10]);
+                        job.sock = outIos;
                             OOBPool->addJob(job);
                             break;
                         }
@@ -2210,6 +2240,11 @@ struct ReadThread
                             PriorityThreadPool::Job job;
                             job.functor = boost::shared_ptr<PriorityThreadPool::Functor>(new BPPHandler::Destroy(fBPPHandler, bs));
                             job.id = fBPPHandler->getUniqueID(bs, ismHdr->Command);
+                        const uint8_t *buf = bs->buf();
+                        uint32_t pos = sizeof(ISMPacketHeader) - 2;
+                        job.stepID = *((uint32_t *) &buf[pos+6]);
+                        job.uniqueID = *((uint32_t *) &buf[pos+10]);
+                        job.sock = outIos;
                             OOBPool->addJob(job);
                             //fBPPHandler->destroyBPP(*bs);
                             break;
@@ -2228,6 +2263,11 @@ struct ReadThread
                             PriorityThreadPool::Job job;
                             job.functor = boost::shared_ptr<PriorityThreadPool::Functor>(new BPPHandler::Abort(fBPPHandler, bs));
                             job.id = fBPPHandler->getUniqueID(bs, ismHdr->Command);
+                        const uint8_t *buf = bs->buf();
+                        uint32_t pos = sizeof(ISMPacketHeader) - 2;
+                        job.stepID = *((uint32_t *) &buf[pos+6]);
+                        job.uniqueID = *((uint32_t *) &buf[pos+10]);
+                        job.sock = outIos;
                             OOBPool->addJob(job);
                             break;
                         }

procmgr/main.cpp

@@ -1688,7 +1688,7 @@ void pingDeviceThread()
                                     processManager.restartProcessType("WriteEngineServer", moduleName);
 
                                     //set module to enable state
-                                    processManager.enableModule(moduleName, oam::AUTO_OFFLINE);
+									processManager.enableModule(moduleName, oam::AUTO_OFFLINE, true);
 
                                     downActiveOAMModule = false;
                                     int retry;
@@ -1784,7 +1784,7 @@ void pingDeviceThread()
                                 }
                                 else
                                     //set module to enable state
-                                    processManager.enableModule(moduleName, oam::AUTO_OFFLINE);
+								    processManager.enableModule(moduleName, oam::AUTO_OFFLINE, true);
 
                                 //restart module processes
                                 int retry = 0;
@@ -2094,7 +2094,7 @@ void pingDeviceThread()
                                 if ( PrimaryUMModuleName == moduleName )
                                     downPrimaryUM = true;
 
-                                // if not disabled and amazon, skip
+								// if disabled, skip
                                 if (opState != oam::AUTO_DISABLED )
                                 {
                                     //Log failure, issue alarm, set moduleOpState
@@ -2140,7 +2140,7 @@ void pingDeviceThread()
                                     if ( ( moduleName.find("pm") == 0 && !amazon && ( DBRootStorageType != "internal") ) ||
                                             ( moduleName.find("pm") == 0 && amazon && downActiveOAMModule ) ||
                                             ( moduleName.find("pm") == 0 && amazon && AmazonPMFailover == "y") )
-                                    {
+										string error;
                                         try
                                         {
                                             log.writeLog(__LINE__, "Call autoMovePmDbroot", LOG_TYPE_DEBUG);
@@ -2158,6 +2158,23 @@ void pingDeviceThread()
                                         {
                                             log.writeLog(__LINE__, "EXCEPTION ERROR on autoMovePmDbroot: Caught unknown exception!", LOG_TYPE_ERROR);
 										}
+										
+										if ( error == oam.itoa(oam::API_DETACH_FAILURE) )
+										{
+										      processManager.setModuleState(moduleName, oam::AUTO_DISABLED);
+
+										      // resume the dbrm
+										      oam.dbrmctl("resume");
+										      log.writeLog(__LINE__, "'dbrmctl resume' done", LOG_TYPE_DEBUG);
+
+										      //enable query stats
+										      dbrm.setSystemQueryReady(true);
+
+										      //set query system state ready
+										      processManager.setQuerySystemState(true);
+										      
+										      break;
+                                        }
                                     }
                                 }
 

procmgr/processmanager.cpp

@@ -3780,7 +3780,7 @@ void ProcessManager::recycleProcess(string module, bool enableModule)
     restartProcessType("ExeMgr");
     sleep(1);
 
-    restartProcessType("mysql");
+	restartProcessType("mysqld");
 
     restartProcessType("WriteEngineServer");
     sleep(1);
@@ -3799,7 +3799,7 @@ void ProcessManager::recycleProcess(string module, bool enableModule)
 * purpose:	Clear the Disable State on a specified module
 *
 ******************************************************************************************/
-int ProcessManager::enableModule(string target, int state)
+int ProcessManager::enableModule(string target, int state, bool failover)
 {
     Oam oam;
     ModuleConfig moduleconfig;
@@ -3839,6 +3839,7 @@ int ProcessManager::enableModule(string target, int state)
         setStandbyModule(newStandbyModule);
 
     //set recycle process
+	if (!failover)
 	      recycleProcess(target);
 
     log.writeLog(__LINE__, "enableModule request for " + target + " completed", LOG_TYPE_DEBUG);
@@ -4647,7 +4648,7 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski
         PMwithUM = "n";
     }
 
-    // If mysql is the processName, then send to modules were ExeMgr is running
+	// If mysqld is the processName, then send to modules were ExeMgr is running
     try
     {
         oam.getProcessStatus(systemprocessstatus);
@@ -4658,7 +4659,7 @@ int ProcessManager::restartProcessType( std::string processName, std::string ski
             if ( systemprocessstatus.processstatus[i].Module == skipModule )
                 continue;
 
-            if ( processName == "mysql" )
+			if ( processName == "mysqld" ) {
             {
                 if ( systemprocessstatus.processstatus[i].ProcessName == "ExeMgr")
                 {
@@ -9813,7 +9814,7 @@ int ProcessManager::OAMParentModuleChange()
                         {
                             log.writeLog(__LINE__, "System Active, restart needed processes", LOG_TYPE_DEBUG);
 
-                            processManager.restartProcessType("mysql");
+							processManager.restartProcessType("mysqld");
                             processManager.restartProcessType("ExeMgr");
                             processManager.restartProcessType("WriteEngineServer");
                             processManager.reinitProcessType("DBRMWorkerNode");
@@ -11013,7 +11014,7 @@ void ProcessManager::stopProcessTypes(bool manualFlag)
     log.writeLog(__LINE__, "stopProcessTypes Called");
 
     //front-end first
-    processManager.stopProcessType("mysql", manualFlag);
+	processManager.stopProcessType("mysqld", manualFlag);
     processManager.stopProcessType("DMLProc", manualFlag);
     processManager.stopProcessType("DDLProc", manualFlag);
     processManager.stopProcessType("ExeMgr", manualFlag);

procmgr/processmanager.h

@@ -309,7 +309,7 @@ public:
     /**
     *@brief Enable a specified module
     */
-    int enableModule(std::string target, int state);
+    int enableModule(std::string target, int state, bool failover = false);
 
     /**
     *@brief Enable a specified module

procmon/main.cpp

@@ -781,11 +781,11 @@ int main(int argc, char** argv)
     if ( ret != 0 )
         log.writeLog(__LINE__, "pthread_create failed, return code = " + oam.itoa(ret), LOG_TYPE_ERROR);
 
-    //mysql status monitor thread
+	//mysqld status monitor thread
-    if ( ( config.ServerInstallType() != oam::INSTALL_COMBINE_DM_UM_PM ) ||
+	if ( config.moduleType() == "um" ||
-            (PMwithUM == "y") )
+		( config.moduleType() == "pm" && config.ServerInstallType() == oam::INSTALL_COMBINE_DM_UM_PM ) ||
+		( config.moduleType() == "pm" && PMwithUM == "y") )
     {
-
         pthread_t mysqlThread;
         ret = pthread_create (&mysqlThread, NULL, (void* (*)(void*)) &mysqlMonitorThread, NULL);
 
@@ -1233,7 +1233,7 @@ static void mysqlMonitorThread(MonitorConfig config)
         catch (...)
         {}
 
-        sleep(10);
+		sleep(5);
     }
 
 }

procmon/processmonitor.cpp

@@ -484,7 +484,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO
                     log.writeLog(__LINE__,  "MSG RECEIVED: Stop process request on " + processName);
                     int requestStatus = API_SUCCESS;
 
-                    // check for mysql
+					// check for mysqld
                     if ( processName == "mysqld" )
                     {
                         try
@@ -553,7 +553,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO
                     msg >> manualFlag;
                     log.writeLog(__LINE__, "MSG RECEIVED: Start process request on: " + processName);
 
-                    // check for mysql
+					// check for mysqld
                     if ( processName == "mysqld" )
                     {
                         try
@@ -684,7 +684,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO
                     log.writeLog(__LINE__,  "MSG RECEIVED: Restart process request on " + processName);
                     int requestStatus = API_SUCCESS;
 
-                    // check for mysql restart
+					// check for mysqld restart
                     if ( processName == "mysqld" )
                     {
                         try
@@ -933,7 +933,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO
                                 log.writeLog(__LINE__, "Error running DBRM clearShm", LOG_TYPE_ERROR);
                         }
 
-                        //stop the mysql daemon
+						//stop the mysqld daemon
                         try
                         {
                             oam.actionMysqlCalpont(MYSQL_STOP);
@@ -1071,13 +1071,13 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO
 
                     system(cmd.c_str());
 
-                    //start the mysql daemon
+					//start the mysqld daemon 
                     try
                     {
                         oam.actionMysqlCalpont(MYSQL_START);
                     }
                     catch (...)
-                    {
+					{	// mysqld didn't start, return with error
                         // mysql didn't start, return with error
                         log.writeLog(__LINE__, "STARTALL: MySQL failed to start, start-module failure", LOG_TYPE_CRITICAL);
 
@@ -1366,7 +1366,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO
                     					//send down notification
                     					oam.sendDeviceNotification(config.moduleName(), MODULE_DOWN);
 
-                    					//stop the mysql daemon and then columnstore
+					//stop the mysqld daemon and then columnstore
                     					try {
                     						oam.actionMysqlCalpont(MYSQL_STOP);
                     					}
@@ -1548,7 +1548,7 @@ void ProcessMonitor::processMessage(messageqcpp::ByteStream msg, messageqcpp::IO
                 }
             }
 
-            // install mysql rpms if being reconfigured as a um
+			// install mysqld rpms if being reconfigured as a um
             if ( reconfigureModuleName.find("um") != string::npos )
             {
                 string cmd = startup::StartUp::installDir() + "/bin/post-mysqld-install >> /tmp/rpminstall";

utils/libmysql_client/libmysql_client.cpp

@@ -120,12 +120,16 @@ int LibMySQL::run(const char* query)
 void LibMySQL::handleMySqlError(const char* errStr, unsigned int errCode)
 {
 	ostringstream oss;
-    oss << errStr << "(" << errCode << ")";
+    if (mysql->getErrno())
-
+    {
-    if (errCode == (unsigned int) - 1)
+        oss << errStr << " (" << mysql->getErrno() << ")";
-        oss << "(null pointer)";
+        oss << " (" << mysql->getErrorMsg() << ")";
+    }
     else
-        oss << "(" << errCode << ")";
+    {
+	    oss << errStr << " (" << errCode << ")";
+		oss << " (unknown)";
+    }
 
     throw logging::IDBExcept(oss.str(), logging::ERR_CROSS_ENGINE_CONNECT);
 

utils/libmysql_client/libmysql_client.h

@@ -71,6 +71,8 @@ public:
     {
         return fErrStr;
     }
+    unsigned int getErrno() { return mysql_errno(fCon); }
+    const char* getErrorMsg() { return mysql_error(fCon); }
 
 private:
     MYSQL*          fCon;

utils/threadpool/prioritythreadpool.cpp

@@ -33,6 +33,8 @@ using namespace logging;
 #include "prioritythreadpool.h"
 using namespace boost;
 
+#include "dbcon/joblist/primitivemsg.h"
+
 namespace threadpool
 {
 
@@ -51,9 +53,9 @@ PriorityThreadPool::PriorityThreadPool(uint targetWeightPerRun, uint highThreads
 
     cout << "started " << highThreads << " high, " << midThreads << " med, " << lowThreads
          << " low.\n";
-    threadCounts[HIGH] = highThreads;
+	defaultThreadCounts[HIGH] = threadCounts[HIGH] = highThreads;
-    threadCounts[MEDIUM] = midThreads;
+	defaultThreadCounts[MEDIUM] = threadCounts[MEDIUM] = midThreads;
-    threadCounts[LOW] = lowThreads;
+	defaultThreadCounts[LOW] = threadCounts[LOW] = lowThreads;
 }
 
 PriorityThreadPool::~PriorityThreadPool()
@@ -68,6 +70,23 @@ void PriorityThreadPool::addJob(const Job& job, bool useLock)
     if (useLock)
         lk.lock();
 
+    // Create any missing threads
+    if (defaultThreadCounts[HIGH] != threadCounts[HIGH])
+    {
+        threads.create_thread(ThreadHelper(this, HIGH));
+        threadCounts[HIGH]++;
+    }
+    if (defaultThreadCounts[MEDIUM] != threadCounts[MEDIUM])
+    {
+        threads.create_thread(ThreadHelper(this, MEDIUM));
+        threadCounts[MEDIUM]++;
+    }
+    if (defaultThreadCounts[LOW] != threadCounts[LOW])
+    {
+        threads.create_thread(ThreadHelper(this, LOW));
+        threadCounts[LOW]++;
+    }
+
     if (job.priority > 66)
         jobQueues[HIGH].push_back(job);
     else if (job.priority > 33)
@@ -113,14 +132,16 @@ void PriorityThreadPool::threadFcn(const Priority preferredQueue) throw()
     vector<bool> reschedule;
     uint32_t rescheduleCount;
     uint32_t queueSize;
+    bool running = false;
 
-    while (!_stop)
+    try
     {
+        while (!_stop) {
 
             mutex::scoped_lock lk(mutex);
 
             queue = pickAQueue(preferredQueue);
-
+            if (jobQueues[queue].empty()) {
         if (jobQueues[queue].empty())
         {
                 newJob.wait(lk);
@@ -137,7 +158,7 @@ void PriorityThreadPool::threadFcn(const Priority preferredQueue) throw()
             //     should leave some to the other threads
 
             while ((weight < weightPerRun) && (!jobQueues[queue].empty())
-                && (runList.size() <= queueSize / 2))
+              && (runList.size() <= queueSize/2)) {
         {
                 runList.push_back(jobQueues[queue].front());
                 jobQueues[queue].pop_front();
@@ -148,28 +169,24 @@ void PriorityThreadPool::threadFcn(const Priority preferredQueue) throw()
 
             reschedule.resize(runList.size());
             rescheduleCount = 0;
-
+            for (i = 0; i < runList.size() && !_stop; i++) {
-        for (i = 0; i < runList.size() && !_stop; i++)
         {
             try
             {
                     reschedule[i] = false;
+                    running = true;
                     reschedule[i] = (*(runList[i].functor))();
-
+                    running = false;
                     if (reschedule[i])
                         rescheduleCount++;
             }
-            catch (std::exception& e)
             {
-                cerr << e.what() << endl;
-            }
-        }
 
             // no real work was done, prevent intensive busy waiting
             if (rescheduleCount == runList.size())
                 usleep(1000);
 
-        if (rescheduleCount > 0)
+            if (rescheduleCount > 0) {
         {
                 lk.lock();
 
@@ -187,6 +204,74 @@ void PriorityThreadPool::threadFcn(const Priority preferredQueue) throw()
 
             runList.clear();
         }
+    }
+    catch (std::exception &ex)
+    {
+        // Log the exception and exit this thread
+        try
+        {
+            threadCounts[queue]--;
+#ifndef NOLOGGING
+            logging::Message::Args args;
+            logging::Message message(5);
+            args.add("threadFcn: Caught exception: ");
+            args.add(ex.what());
+
+            message.format( args );
+
+            logging::LoggingID lid(22);
+            logging::MessageLog ml(lid);
+
+            ml.logErrorMessage( message );
+#endif
+            if (running)
+                sendErrorMsg(runList[i].uniqueID, runList[i].stepID, runList[i].sock);
+        }
+        catch (...)
+        {
+        }
+    }
+    catch (...)
+    {
+
+        // Log the exception and exit this thread
+        try
+        {
+            threadCounts[queue]--;
+#ifndef NOLOGGING
+            logging::Message::Args args;
+            logging::Message message(6);
+            args.add("threadFcn: Caught unknown exception!");
+
+            message.format( args );
+
+            logging::LoggingID lid(22);
+            logging::MessageLog ml(lid);
+
+            ml.logErrorMessage( message );
+#endif
+            if (running)
+                sendErrorMsg(runList[i].uniqueID, runList[i].stepID, runList[i].sock);
+        }
+        catch (...)
+        {
+        }
+    }
+}
+
+void PriorityThreadPool::sendErrorMsg(uint32_t id, uint32_t step, primitiveprocessor::SP_UM_IOSOCK sock)
+{
+	ISMPacketHeader ism;
+	PrimitiveHeader ph = {0};
+
+	ism.Status =  logging::primitiveServerErr;
+	ph.UniqueID = id;
+	ph.StepID = step;
+	ByteStream msg(sizeof(ISMPacketHeader) + sizeof(PrimitiveHeader));
+	msg.append((uint8_t *) &ism, sizeof(ism));
+	msg.append((uint8_t *) &ph, sizeof(ph));
+
+	sock->write(msg);
 }
 
 void PriorityThreadPool::stop()

utils/threadpool/prioritythreadpool.h

@@ -36,6 +36,7 @@
 #include <boost/shared_ptr.hpp>
 #include <boost/function.hpp>
 #include "../winport/winport.h"
+#include "primitives/primproc/umsocketselector.h"
 
 namespace threadpool
 {
@@ -62,6 +63,9 @@ public:
         uint32_t weight;
         uint32_t priority;
         uint32_t id;
+        uint32_t uniqueID;
+        uint32_t stepID;
+        primitiveprocessor::SP_UM_IOSOCK sock;
     };
 
     enum Priority
@@ -112,9 +116,11 @@ private:
 
     Priority pickAQueue(Priority preference);
     void threadFcn(const Priority preferredQueue) throw();
+    void sendErrorMsg(uint32_t id, uint32_t step, primitiveprocessor::SP_UM_IOSOCK sock);
 
     std::list<Job> jobQueues[3];  // higher indexes = higher priority
     uint32_t threadCounts[3];
+    uint32_t defaultThreadCounts[3];
     boost::mutex mutex;
     boost::condition newJob;
     boost::thread_group threads;

writeengine/wrapper/writeengine.cpp

@@ -2246,7 +2246,6 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
         if (it != aColExtsInfo.end()) //update hwm info
         {
             oldHwm = it->hwm;
-        }
 
         // save hwm for the old extent
         colWidth = colStructList[i].colWidth;
@@ -2272,6 +2271,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
         else
             return ERR_INVALID_PARAM;
 
+        }
         //update hwm for the new extent
         if (newExtent)
         {
@@ -2285,7 +2285,7 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
 
                 it++;
             }
-
+        colWidth = newColStructList[i].colWidth;
             succFlag = colOp->calculateRowId(lastRidNew, BYTE_PER_BLOCK / colWidth, colWidth, curFbo, curBio);
 
             if (succFlag)
@@ -2356,6 +2356,9 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
                                                       curFbo));
                     }
                 }
+            else
+                return ERR_INVALID_PARAM;
+        }
             }
 
             // If we create a new extent for this batch
@@ -2376,7 +2379,8 @@ int WriteEngineWrapper::insertColumnRecsBinary(const TxnID& txnid,
                                                       curFbo));
                     }
                 }
-            }
+            else
+                return ERR_INVALID_PARAM;
         }
 
         if (lbids.size() > 0)
@@ -5132,7 +5136,7 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid,
         bool versioning)
 {
     int            rc = 0;
-    void*          valArray;
+   void*          valArray = NULL;
     string         segFile;
     Column         curCol;
     ColStructList::size_type  totalColumn;
@@ -5158,18 +5162,19 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid,
         totalRow2 = 0;
     }
 
-    valArray = malloc(sizeof(uint64_t) * totalRow1);
+    // It is possible totalRow1 is zero but totalRow2 has values
-
+    if ((totalRow1 == 0) && (totalRow2 == 0))
-    if (totalRow1 == 0)
         return rc;
 
     TableMetaData* aTbaleMetaData = TableMetaData::makeTableMetaData(tableOid);
-
+    if (totalRow1)
+    {
+        valArray = malloc(sizeof(uint64_t) * totalRow1);
         for (i = 0; i < totalColumn; i++)
         {
              //@Bug 2205 Check if all rows go to the new extent
             //Write the first batch
-        RID* firstPart = rowIdArray;
+            RID * firstPart = rowIdArray;
             ColumnOp* colOp = m_colOp[op(colStructList[i].fCompressionType)];
 
             // set params
@@ -5194,7 +5199,7 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid,
             if (it == aColExtsInfo.end()) //add this one to the list
             {
                 ColExtInfo aExt;
-            aExt.dbRoot = colStructList[i].fColDbRoot;
+                aExt.dbRoot =colStructList[i].fColDbRoot;
                 aExt.partNum = colStructList[i].fColPartition;
                 aExt.segNum = colStructList[i].fColSegment;
                 aExt.compType = colStructList[i].fCompressionType;
@@ -5214,7 +5219,7 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid,
             {
                     rc = processVersionBuffer(curCol.dataFile.pFile, txnid, colStructList[i],
                                       colStructList[i].colWidth, totalRow1, firstPart, rangeList);
-
+                if (rc != NO_ERROR) {
             if (rc != NO_ERROR)
             {
                     if (colStructList[i].fCompressionType == 0)
@@ -5236,7 +5241,7 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid,
 
             for (size_t j = 0; j < totalRow1; j++)
             {
-            uint64_t curValue = colValueList[((totalRow1 + totalRow2) * i) + j];
+                uint64_t curValue = colValueList[((totalRow1 + totalRow2)*i) + j];
 
                 switch (colStructList[i].colType)
                 {
@@ -5299,6 +5304,7 @@ int WriteEngineWrapper::writeColumnRecBinary(const TxnID& txnid,
             free(valArray);
             valArray = NULL;
         }
+    }
 
     // MCOL-1176 - Write second extent
     if (totalRow2)