From 82ee220cfa7aaad0ff1eccd0740e8bb1a0249b3c Mon Sep 17 00:00:00 2001 From: "brian@zim.(none)" <> Date: Tue, 6 Mar 2007 17:14:59 -0800 Subject: [PATCH 01/28] Fixed buffer overflow cases (should not be possible to do...) Fixed for autoincrement support/GUID --- client/client_priv.h | 2 + client/mysqlslap.c | 373 +++++++++++++++++++++++++++++++++--- mysql-test/t/mysqlslap.test | 14 ++ 3 files changed, 359 insertions(+), 30 deletions(-) diff --git a/client/client_priv.h b/client/client_priv.h index 646619f62b1..c8cf68bfd13 100644 --- a/client/client_priv.h +++ b/client/client_priv.h @@ -58,6 +58,8 @@ enum options_client OPT_TZ_UTC, OPT_AUTO_CLOSE, OPT_CREATE_SLAP_SCHEMA, OPT_SLAP_CSV, OPT_SLAP_CREATE_STRING, OPT_SLAP_AUTO_GENERATE_SQL_LOAD_TYPE, OPT_SLAP_AUTO_GENERATE_WRITE_NUM, + OPT_SLAP_AUTO_GENERATE_ADD_AUTO, + OPT_SLAP_AUTO_GENERATE_GUID_PRIMARY, OPT_MYSQL_REPLACE_INTO, OPT_BASE64_OUTPUT, OPT_SERVER_ID, OPT_FIX_TABLE_NAMES, OPT_FIX_DB_NAMES, OPT_SSL_VERIFY_SERVER_CERT, OPT_DEBUG_INFO, OPT_COLUMN_TYPES diff --git a/client/mysqlslap.c b/client/mysqlslap.c index ad2c8685ba1..047193495d2 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -71,9 +71,16 @@ TODO: #define SHOW_VERSION "0.9" -#define HUGE_STRING_LENGTH 8096 +#define HUGE_STRING_LENGTH 8196 #define RAND_STRING_SIZE 126 +/* Types */ +#define SELECT_TYPE 0 +#define UPDATE_TYPE 1 +#define INSERT_TYPE 2 +#define UPDATE_TYPE_REQUERIES_PREFIX 3 +#define CREATE_TABLE_TYPE 4 + #include "client_priv.h" #ifdef HAVE_LIBPTHREAD #include @@ -107,6 +114,9 @@ static char *shared_memory_base_name=0; static char **defaults_argv; +char *primary_keys; +unsigned long long primary_keys_number_of; + static char *host= NULL, *opt_password= NULL, *user= NULL, *user_supplied_query= NULL, *default_engine= NULL, @@ -127,6 +137,8 @@ static my_bool opt_slave; static my_bool opt_compress= FALSE, tty_password= FALSE, opt_silent= FALSE, + auto_generate_sql_autoincrement= FALSE, + auto_generate_sql_guid_primary= FALSE, auto_generate_sql= FALSE; const char *auto_generate_sql_type= "mixed"; @@ -159,6 +171,7 @@ typedef struct statement statement; struct statement { char *string; size_t length; + unsigned char type; statement *next; }; @@ -206,7 +219,10 @@ static int drop_schema(MYSQL *mysql, const char *db); uint get_random_string(char *buf); static statement *build_table_string(void); static statement *build_insert_string(void); +static statement *build_update_string(void); static statement *build_query_string(void); +static int generate_primary_key_list(MYSQL *mysql); +static int drop_primary_key_list(void); static int create_schema(MYSQL *mysql, const char *db, statement *stmt, statement *engine_stmt); static int run_scheduler(stats *sptr, statement *stmts, uint concur, @@ -343,6 +359,13 @@ int main(int argc, char **argv) /* First we create */ if (create_statements) create_schema(&mysql, create_schema_string, create_statements, eptr); + + /* + If we generated GUID we need to build a list of them from creation that + we can later use. + */ + if (auto_generate_sql_guid_primary) + generate_primary_key_list(&mysql); run_scheduler(sptr, query_statements, *current, client_limit); } @@ -359,6 +382,10 @@ int main(int argc, char **argv) if (!opt_preserve) drop_schema(&mysql, create_schema_string); + + if (auto_generate_sql_guid_primary) + drop_primary_key_list(); + } while (eptr ? (eptr= eptr->next) : 0); if (!opt_only_print) @@ -397,8 +424,18 @@ static struct my_option my_long_options[] = "Generate SQL where not supplied by file or command line.", (gptr*) &auto_generate_sql, (gptr*) &auto_generate_sql, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"auto-generate-sql-add-autoincrement", OPT_SLAP_AUTO_GENERATE_ADD_AUTO, + "Add autoincrement to auto-generated tables.", + (gptr*) &auto_generate_sql_autoincrement, + (gptr*) &auto_generate_sql_autoincrement, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"auto-generate-sql-guid-primary", OPT_SLAP_AUTO_GENERATE_GUID_PRIMARY, + "Add GUID based primary keys to auto-generated tables.", + (gptr*) &auto_generate_sql_guid_primary, + (gptr*) &auto_generate_sql_guid_primary, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"auto-generate-sql-load-type", OPT_SLAP_AUTO_GENERATE_SQL_LOAD_TYPE, - "Load types are mixed, write, or read. Default is mixed\n", + "Load types are mixed, update, write, or read. Default is mixed\n", (gptr*) &auto_generate_sql_type, (gptr*) &auto_generate_sql_type, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"auto-generate-sql-write-number", OPT_SLAP_AUTO_GENERATE_WRITE_NUM, @@ -614,7 +651,7 @@ get_random_string(char *buf) static statement * build_table_string(void) { - char buf[512]; + char buf[HUGE_STRING_LENGTH]; int col_count; statement *ptr; DYNAMIC_STRING table_string; @@ -626,22 +663,63 @@ build_table_string(void) init_dynamic_string(&table_string, "", 1024, 1024); dynstr_append(&table_string, "CREATE TABLE `t1` ("); - for (col_count= 1; col_count <= num_int_cols; col_count++) + + if (auto_generate_sql_autoincrement) { - sprintf(buf, "intcol%d INT(32)", col_count); + if (snprintf(buf, HUGE_STRING_LENGTH, "id serial") > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in create table\n"); + exit(1); + } dynstr_append(&table_string, buf); - if (col_count < num_int_cols || num_char_cols > 0) + if (num_int_cols || num_char_cols) dynstr_append(&table_string, ","); } - for (col_count= 1; col_count <= num_char_cols; col_count++) + + if (auto_generate_sql_guid_primary) { - sprintf(buf, "charcol%d VARCHAR(128)", col_count); + if (snprintf(buf, HUGE_STRING_LENGTH, "id varchar(32) primary key") > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in create table\n"); + exit(1); + } dynstr_append(&table_string, buf); - if (col_count < num_char_cols) + if (num_int_cols || num_char_cols) dynstr_append(&table_string, ","); } + + if (num_int_cols) + for (col_count= 1; col_count <= num_int_cols; col_count++) + { + if (snprintf(buf, HUGE_STRING_LENGTH, "intcol%d INT(32)", col_count) + > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in create table\n"); + exit(1); + } + dynstr_append(&table_string, buf); + + if (col_count < num_int_cols || num_char_cols > 0) + dynstr_append(&table_string, ","); + } + + if (num_char_cols) + for (col_count= 1; col_count <= num_char_cols; col_count++) + { + if (snprintf(buf, HUGE_STRING_LENGTH, "charcol%d VARCHAR(128)", col_count) + > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating table\n"); + exit(1); + } + dynstr_append(&table_string, buf); + + if (col_count < num_char_cols) + dynstr_append(&table_string, ","); + } + dynstr_append(&table_string, ")"); ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)); ptr->string = (char *)my_malloc(table_string.length+1, MYF(MY_WME)); @@ -651,6 +729,94 @@ build_table_string(void) DBUG_RETURN(ptr); } +/* + build_update_string() + + This function builds insert statements when the user opts to not supply + an insert file or string containing insert data +*/ +static statement * +build_update_string(void) +{ + char buf[HUGE_STRING_LENGTH]; + int col_count; + statement *ptr; + DYNAMIC_STRING update_string; + DBUG_ENTER("build_update_string"); + + init_dynamic_string(&update_string, "", 1024, 1024); + + dynstr_append(&update_string, "UPDATE t1 SET "); + + if (num_int_cols) + for (col_count= 1; col_count <= num_int_cols; col_count++) + { + if (snprintf(buf, HUGE_STRING_LENGTH, "intcol%d = %ld", col_count, + random()) > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating update\n"); + exit(1); + } + dynstr_append(&update_string, buf); + + if (col_count < num_int_cols || num_char_cols > 0) + dynstr_append_mem(&update_string, ",", 1); + } + + if (num_char_cols) + for (col_count= 1; col_count <= num_char_cols; col_count++) + { + char rand_buffer[RAND_STRING_SIZE]; + int buf_len= get_random_string(rand_buffer); + + if (snprintf(buf, HUGE_STRING_LENGTH, "charcol%d = '%.*s'", col_count, + buf_len, rand_buffer) + > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating update\n"); + exit(1); + } + dynstr_append(&update_string, buf); + + if (col_count < num_char_cols) + dynstr_append_mem(&update_string, ",", 1); + } + + if (auto_generate_sql_autoincrement) + { + if (snprintf(buf, HUGE_STRING_LENGTH, " WHERE id = %ld", + (long int)(random() % auto_generate_sql_number) ) + > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating update_string\n"); + exit(1); + } + dynstr_append(&update_string, buf); + } + + if (auto_generate_sql_guid_primary) + { + if (snprintf(buf, HUGE_STRING_LENGTH, " WHERE id = ") > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating update_string\n"); + exit(1); + } + dynstr_append(&update_string, buf); + } + + + ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)); + ptr->string= (char *)my_malloc(update_string.length+1, MYF(MY_WME)); + ptr->length= update_string.length+1; + if (auto_generate_sql_guid_primary) + ptr->type= UPDATE_TYPE_REQUERIES_PREFIX ; + else + ptr->type= UPDATE_TYPE; + strmov(ptr->string, update_string.str); + dynstr_free(&update_string); + DBUG_RETURN(ptr); +} + /* build_insert_string() @@ -661,7 +827,7 @@ build_table_string(void) static statement * build_insert_string(void) { - char buf[RAND_STRING_SIZE]; + char buf[HUGE_STRING_LENGTH]; int col_count; statement *ptr; DYNAMIC_STRING insert_string; @@ -669,30 +835,66 @@ build_insert_string(void) init_dynamic_string(&insert_string, "", 1024, 1024); - dynstr_append_mem(&insert_string, "INSERT INTO t1 VALUES (", 23); - for (col_count= 1; col_count <= num_int_cols; col_count++) + dynstr_append(&insert_string, "INSERT INTO t1 VALUES ("); + + if (auto_generate_sql_autoincrement) { - sprintf(buf, "%ld", random()); + if (snprintf(buf, HUGE_STRING_LENGTH, "NULL") > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating insert\n"); + exit(1); + } dynstr_append(&insert_string, buf); - if (col_count < num_int_cols || num_char_cols > 0) - dynstr_append_mem(&insert_string, ",", 1); + if (num_int_cols || num_char_cols) + dynstr_append(&insert_string, ","); } - for (col_count= 1; col_count <= num_char_cols; col_count++) - { - int buf_len= get_random_string(buf); - dynstr_append_mem(&insert_string, "'", 1); - dynstr_append_mem(&insert_string, buf, buf_len); - dynstr_append_mem(&insert_string, "'", 1); - if (col_count < num_char_cols) - dynstr_append_mem(&insert_string, ",", 1); + if (auto_generate_sql_guid_primary) + { + if (snprintf(buf, HUGE_STRING_LENGTH, "uuid()") > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in create table\n"); + exit(1); + } + dynstr_append(&insert_string, buf); + + if (num_int_cols || num_char_cols) + dynstr_append(&insert_string, ","); } + + if (num_int_cols) + for (col_count= 1; col_count <= num_int_cols; col_count++) + { + if (snprintf(buf, HUGE_STRING_LENGTH, "%ld", random()) > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating insert\n"); + exit(1); + } + dynstr_append(&insert_string, buf); + + if (col_count < num_int_cols || num_char_cols > 0) + dynstr_append_mem(&insert_string, ",", 1); + } + + if (num_char_cols) + for (col_count= 1; col_count <= num_char_cols; col_count++) + { + int buf_len= get_random_string(buf); + dynstr_append_mem(&insert_string, "'", 1); + dynstr_append_mem(&insert_string, buf, buf_len); + dynstr_append_mem(&insert_string, "'", 1); + + if (col_count < num_char_cols) + dynstr_append_mem(&insert_string, ",", 1); + } + dynstr_append_mem(&insert_string, ")", 1); ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)); ptr->string= (char *)my_malloc(insert_string.length+1, MYF(MY_WME)); ptr->length= insert_string.length+1; + ptr->type= INSERT_TYPE; strmov(ptr->string, insert_string.str); dynstr_free(&insert_string); DBUG_RETURN(ptr); @@ -708,7 +910,7 @@ build_insert_string(void) static statement * build_query_string(void) { - char buf[512]; + char buf[HUGE_STRING_LENGTH]; int col_count; statement *ptr; static DYNAMIC_STRING query_string; @@ -719,7 +921,12 @@ build_query_string(void) dynstr_append_mem(&query_string, "SELECT ", 7); for (col_count= 1; col_count <= num_int_cols; col_count++) { - sprintf(buf, "intcol%d", col_count); + if (snprintf(buf, HUGE_STRING_LENGTH, "intcol%d", col_count) + > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating select\n"); + exit(1); + } dynstr_append(&query_string, buf); if (col_count < num_int_cols || num_char_cols > 0) @@ -728,7 +935,12 @@ build_query_string(void) } for (col_count= 1; col_count <= num_char_cols; col_count++) { - sprintf(buf, "charcol%d", col_count); + if (snprintf(buf, HUGE_STRING_LENGTH, "charcol%d", col_count) + > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating select\n"); + exit(1); + } dynstr_append(&query_string, buf); if (col_count < num_char_cols) @@ -739,6 +951,7 @@ build_query_string(void) ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)); ptr->string= (char *)my_malloc(query_string.length+1, MYF(MY_WME)); ptr->length= query_string.length+1; + ptr->type= SELECT_TYPE; strmov(ptr->string, query_string.str); dynstr_free(&query_string); DBUG_RETURN(ptr); @@ -773,6 +986,14 @@ get_options(int *argc,char ***argv) exit(1); } + if (auto_generate_sql && auto_generate_sql_guid_primary && auto_generate_sql_autoincrement) + { + fprintf(stderr, + "%s: Either auto-generate-sql-guid-primary or auto-generate-sql-add-autoincrement can be used!\n", + my_progname); + exit(1); + } + parse_comma(concurrency_str ? concurrency_str : "1", &concurrency); if (lock_directory) @@ -836,6 +1057,23 @@ get_options(int *argc,char ***argv) ptr_statement->next= build_insert_string(); } } + else if (auto_generate_sql_type[0] == 'u') + { + for (ptr_statement= create_statements, x= 0; + x < auto_generate_sql_number; + x++, ptr_statement= ptr_statement->next) + { + ptr_statement->next= build_insert_string(); + } + + query_statements= build_update_string(); + for (ptr_statement= query_statements, x= 0; + x < auto_generate_sql_number; + x++, ptr_statement= ptr_statement->next) + { + ptr_statement->next= build_update_string(); + } + } else /* Mixed mode is default */ { int coin= 0; @@ -943,6 +1181,59 @@ static int run_query(MYSQL *mysql, const char *query, int len) } +static int +generate_primary_key_list(MYSQL *mysql) +{ + char query[HUGE_STRING_LENGTH]; + int len; + MYSQL_RES *result; + MYSQL_ROW row; + char *ptr; + DBUG_ENTER("create_schema"); + + len= snprintf(query, HUGE_STRING_LENGTH, "SELECT id from t1"); + + if (run_query(mysql, query, len)) + { + fprintf(stderr,"%s: Cannot select GUID primary keys. (%s)\n", my_progname, + mysql_error(mysql)); + exit(1); + } + + if (opt_only_print) + { + primary_keys_number_of= 1; + primary_keys= (char *)my_malloc(sizeof(char) * primary_keys_number_of * 32, + MYF(MY_ZEROFILL)); + memcpy(primary_keys, "796c4422-1d94-102a-9d6d-00e0812d", 32); + DBUG_RETURN(0); + } + result= mysql_store_result(mysql); + DBUG_ASSERT(result); + primary_keys_number_of= mysql_num_rows(result); + /* + We know the GUID are 32 characters in length, so we preallocate. + */ + primary_keys= (char *)my_malloc(sizeof(char) * primary_keys_number_of * 32, + MYF(MY_ZEROFILL)); + ptr= primary_keys; + while ((row = mysql_fetch_row(result))) + { + memcpy(ptr, row[0], 32); + ptr+= 32; + } + DBUG_ASSERT(ptr == primary_keys + (primary_keys_number_of * 32)); + mysql_free_result(result); + + DBUG_RETURN(0); +} + +static int +drop_primary_key_list(void) +{ + my_free(primary_keys, MYF(0)); + return 0; +} static int create_schema(MYSQL *mysql, const char *db, statement *stmt, @@ -1219,11 +1510,33 @@ run_task(thread_context *con) limit_not_met: for (ptr= con->stmt; ptr && ptr->length; ptr= ptr->next) { - if (run_query(mysql, ptr->string, ptr->length)) + /* + We have to execute differently based on query type. This should become a function. + */ + if (ptr->type == UPDATE_TYPE_REQUERIES_PREFIX) { - fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", - my_progname, (uint)ptr->length, ptr->string, mysql_error(mysql)); - goto end; + char buffer[HUGE_STRING_LENGTH]; + char *key= primary_keys + (random() % primary_keys_number_of); + int length; + + length= snprintf(buffer, HUGE_STRING_LENGTH, "%.*s '%.*s'", + (int)ptr->length, ptr->string, 32, key); + + if (run_query(mysql, buffer, length)) + { + fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", + my_progname, (uint)length, buffer, mysql_error(mysql)); + goto end; + } + } + else + { + if (run_query(mysql, ptr->string, ptr->length)) + { + fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", + my_progname, (uint)ptr->length, ptr->string, mysql_error(mysql)); + goto end; + } } if (mysql_field_count(mysql)) { diff --git a/mysql-test/t/mysqlslap.test b/mysql-test/t/mysqlslap.test index 01add0d7da8..c98da4b4a0f 100644 --- a/mysql-test/t/mysqlslap.test +++ b/mysql-test/t/mysqlslap.test @@ -14,3 +14,17 @@ --exec $MYSQL_SLAP --silent --concurrency=5 --iterations=20 --delimiter=";" --query="select * from t1;select * from t2" --create="CREATE TABLE t1 (id int, name varchar(64)); create table t2(foo1 varchar(32), foo2 varchar(32)); INSERT INTO t1 VALUES (1, 'This is a test'); insert into t2 values ('test', 'test2')" --exec $MYSQL_SLAP --silent --concurrency=5 --iterations=20 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --create-schema=test_env + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=20 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --create-schema=test_env --auto-generate-sql-add-autoincrement + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-add-autoincrement + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-add-autoincrement --auto-generate-sql-load-type=update + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-add-autoincrement --auto-generate-sql-load-type=read + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-add-autoincrement --auto-generate-sql-load-type=write + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-add-autoincrement --auto-generate-sql-load-type=mixed + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --auto-generate-sql-load-type=update From 8623103dfc879baa7e4f3b813ee5fb7ad8f755b0 Mon Sep 17 00:00:00 2001 From: "brian@zim.(none)" <> Date: Mon, 12 Mar 2007 10:25:11 -0700 Subject: [PATCH 02/28] Cleaned up memory allocation so that in all cases of malloc failure the application ends execution. Default values for auto-generate have been made available for adjusting. Key test for rights now works with either type of keys. --- client/client_priv.h | 3 + client/mysqlslap.c | 420 +++++++++++++++++++++++++----------- mysql-test/t/mysqlslap.test | 4 + 3 files changed, 298 insertions(+), 129 deletions(-) diff --git a/client/client_priv.h b/client/client_priv.h index c8cf68bfd13..00ef1a79d96 100644 --- a/client/client_priv.h +++ b/client/client_priv.h @@ -60,6 +60,9 @@ enum options_client OPT_SLAP_AUTO_GENERATE_SQL_LOAD_TYPE, OPT_SLAP_AUTO_GENERATE_WRITE_NUM, OPT_SLAP_AUTO_GENERATE_ADD_AUTO, OPT_SLAP_AUTO_GENERATE_GUID_PRIMARY, + OPT_SLAP_AUTO_GENERATE_EXECUTE_QUERIES, + OPT_SLAP_AUTO_GENERATE_UNIQUE_WRITE_NUM, + OPT_SLAP_AUTO_GENERATE_UNIQUE_QUERY_NUM, OPT_MYSQL_REPLACE_INTO, OPT_BASE64_OUTPUT, OPT_SERVER_ID, OPT_FIX_TABLE_NAMES, OPT_FIX_DB_NAMES, OPT_SSL_VERIFY_SERVER_CERT, OPT_DEBUG_INFO, OPT_COLUMN_TYPES diff --git a/client/mysqlslap.c b/client/mysqlslap.c index 047193495d2..1ec8d124a48 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -78,8 +78,9 @@ TODO: #define SELECT_TYPE 0 #define UPDATE_TYPE 1 #define INSERT_TYPE 2 -#define UPDATE_TYPE_REQUERIES_PREFIX 3 +#define UPDATE_TYPE_REQUIRES_PREFIX 3 #define CREATE_TABLE_TYPE 4 +#define SELECT_TYPE_REQUIRES_PREFIX 5 #include "client_priv.h" #ifdef HAVE_LIBPTHREAD @@ -114,7 +115,7 @@ static char *shared_memory_base_name=0; static char **defaults_argv; -char *primary_keys; +char **primary_keys; unsigned long long primary_keys_number_of; static char *host= NULL, *opt_password= NULL, *user= NULL, @@ -145,9 +146,12 @@ const char *auto_generate_sql_type= "mixed"; static unsigned long connect_flags= CLIENT_MULTI_RESULTS; static int verbose, num_int_cols, num_char_cols, delimiter_length; -static int iterations; +static unsigned int iterations; static char *default_charset= (char*) MYSQL_DEFAULT_CHARSET_NAME; static ulonglong actual_queries= 0; +static ulonglong auto_actual_queries; +static ulonglong auto_generate_sql_unique_write_number; +static ulonglong auto_generate_sql_unique_query_number; static ulonglong num_of_query; static ulonglong auto_generate_sql_number; const char *concurrency_str= NULL; @@ -162,7 +166,7 @@ static uint opt_protocol= 0; static int get_options(int *argc,char ***argv); static uint opt_mysql_port= 0; -static uint opt_use_threads; +static my_bool opt_use_threads; static const char *load_default_groups[]= { "mysqlslap","client",0 }; @@ -220,8 +224,8 @@ uint get_random_string(char *buf); static statement *build_table_string(void); static statement *build_insert_string(void); static statement *build_update_string(void); -static statement *build_query_string(void); -static int generate_primary_key_list(MYSQL *mysql); +static statement * build_select_string(my_bool key); +static int generate_primary_key_list(MYSQL *mysql, statement *engine_stmt); static int drop_primary_key_list(void); static int create_schema(MYSQL *mysql, const char *db, statement *stmt, statement *engine_stmt); @@ -262,7 +266,7 @@ static int gettimeofday(struct timeval *tp, void *tzp) int main(int argc, char **argv) { MYSQL mysql; - int x; + unsigned int x; unsigned long long client_limit; statement *eptr; @@ -333,16 +337,22 @@ int main(int argc, char **argv) uint *current; conclusions conclusion; + if (verbose >= 2) + printf("Starting Concurrency Test\n"); + for (current= concurrency; current && *current; current++) { stats *head_sptr; stats *sptr; - head_sptr= (stats *)my_malloc(sizeof(stats) * iterations, MYF(MY_ZEROFILL)); + head_sptr= (stats *)my_malloc(sizeof(stats) * iterations, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); bzero(&conclusion, sizeof(conclusions)); - if (num_of_query) + if (auto_actual_queries) + client_limit= auto_actual_queries; + else if (num_of_query) client_limit= num_of_query / *current; else client_limit= actual_queries; @@ -356,6 +366,7 @@ int main(int argc, char **argv) */ if (!opt_preserve) drop_schema(&mysql, create_schema_string); + /* First we create */ if (create_statements) create_schema(&mysql, create_schema_string, create_statements, eptr); @@ -364,12 +375,21 @@ int main(int argc, char **argv) If we generated GUID we need to build a list of them from creation that we can later use. */ - if (auto_generate_sql_guid_primary) - generate_primary_key_list(&mysql); + if (verbose >= 2) + printf("Generating primary key list\n"); + if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) + generate_primary_key_list(&mysql, eptr); run_scheduler(sptr, query_statements, *current, client_limit); + + /* We are finished with this run */ + if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) + drop_primary_key_list(); } + if (verbose >= 2) + printf("Generating stats\n"); + generate_stats(&conclusion, eptr, head_sptr); if (!opt_silent) @@ -377,15 +397,12 @@ int main(int argc, char **argv) if (opt_csv_str) print_conclusions_csv(&conclusion); - my_free((byte *)head_sptr, MYF(0)); + my_free((gptr)head_sptr, MYF(0)); } if (!opt_preserve) drop_schema(&mysql, create_schema_string); - if (auto_generate_sql_guid_primary) - drop_primary_key_list(); - } while (eptr ? (eptr= eptr->next) : 0); if (!opt_only_print) @@ -397,9 +414,9 @@ int main(int argc, char **argv) /* now free all the strings we created */ if (opt_password) - my_free(opt_password, MYF(0)); + my_free((gptr)opt_password, MYF(0)); - my_free((byte *)concurrency, MYF(0)); + my_free((gptr)concurrency, MYF(0)); statement_cleanup(create_statements); statement_cleanup(engine_statements); @@ -407,7 +424,7 @@ int main(int argc, char **argv) #ifdef HAVE_SMEM if (shared_memory_base_name) - my_free(shared_memory_base_name,MYF(MY_ALLOW_ZERO_PTR)); + my_free((gptr)shared_memory_base_name, MYF(MY_ALLOW_ZERO_PTR)); #endif free_defaults(defaults_argv); my_end(0); @@ -429,6 +446,10 @@ static struct my_option my_long_options[] = (gptr*) &auto_generate_sql_autoincrement, (gptr*) &auto_generate_sql_autoincrement, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"auto-generate-sql-execute-number", OPT_SLAP_AUTO_GENERATE_EXECUTE_QUERIES, + "Set this number to generate a set number of queries to run.\n", + (gptr*) &auto_actual_queries, (gptr*) &auto_actual_queries, + 0, GET_ULL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"auto-generate-sql-guid-primary", OPT_SLAP_AUTO_GENERATE_GUID_PRIMARY, "Add GUID based primary keys to auto-generated tables.", (gptr*) &auto_generate_sql_guid_primary, @@ -438,6 +459,18 @@ static struct my_option my_long_options[] = "Load types are mixed, update, write, or read. Default is mixed\n", (gptr*) &auto_generate_sql_type, (gptr*) &auto_generate_sql_type, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"auto-generate-sql-unique-query-number", + OPT_SLAP_AUTO_GENERATE_UNIQUE_QUERY_NUM, + "Number of unique queries auto tests", + (gptr*) &auto_generate_sql_unique_query_number, + (gptr*) &auto_generate_sql_unique_query_number, + 0, GET_ULL, REQUIRED_ARG, 10, 0, 0, 0, 0, 0}, + {"auto-generate-sql-unique-write-number", + OPT_SLAP_AUTO_GENERATE_UNIQUE_WRITE_NUM, + "Number of unique queries for auto-generate-sql-write-number", + (gptr*) &auto_generate_sql_unique_write_number, + (gptr*) &auto_generate_sql_unique_write_number, + 0, GET_ULL, REQUIRED_ARG, 10, 0, 0, 0, 0, 0}, {"auto-generate-sql-write-number", OPT_SLAP_AUTO_GENERATE_WRITE_NUM, "Number of rows to insert to used in read and write loads (default is 100).\n", (gptr*) &auto_generate_sql_number, (gptr*) &auto_generate_sql_number, @@ -532,7 +565,7 @@ static struct my_option my_long_options[] = {"use-threads", OPT_USE_THREADS, "Use pthread calls instead of fork() calls (default on Windows)", (gptr*) &opt_use_threads, (gptr*) &opt_use_threads, 0, - GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, #ifndef DONT_ALLOW_USER_CHANGE {"user", 'u', "User for login if not current user.", (gptr*) &user, (gptr*) &user, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -589,7 +622,7 @@ get_one_option(int optid, const struct my_option *opt __attribute__((unused)), if (argument) { char *start= argument; - my_free(opt_password,MYF(MY_ALLOW_ZERO_PTR)); + my_free((gptr)opt_password, MYF(MY_ALLOW_ZERO_PTR)); opt_password= my_strdup(argument,MYF(MY_FAE)); while (*argument) *argument++= 'x'; /* Destroy argument */ if (*start) @@ -721,8 +754,10 @@ build_table_string(void) } dynstr_append(&table_string, ")"); - ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)); - ptr->string = (char *)my_malloc(table_string.length+1, MYF(MY_WME)); + ptr= (statement *)my_malloc(sizeof(statement), + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); + ptr->string = (char *)my_malloc(table_string.length+1, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); ptr->length= table_string.length+1; strmov(ptr->string, table_string.str); dynstr_free(&table_string); @@ -782,19 +817,7 @@ build_update_string(void) dynstr_append_mem(&update_string, ",", 1); } - if (auto_generate_sql_autoincrement) - { - if (snprintf(buf, HUGE_STRING_LENGTH, " WHERE id = %ld", - (long int)(random() % auto_generate_sql_number) ) - > HUGE_STRING_LENGTH) - { - fprintf(stderr, "Memory Allocation error in creating update_string\n"); - exit(1); - } - dynstr_append(&update_string, buf); - } - - if (auto_generate_sql_guid_primary) + if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) { if (snprintf(buf, HUGE_STRING_LENGTH, " WHERE id = ") > HUGE_STRING_LENGTH) { @@ -805,11 +828,14 @@ build_update_string(void) } - ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)); - ptr->string= (char *)my_malloc(update_string.length+1, MYF(MY_WME)); + ptr= (statement *)my_malloc(sizeof(statement), + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); + + ptr->string= (char *)my_malloc(update_string.length + 1, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); ptr->length= update_string.length+1; - if (auto_generate_sql_guid_primary) - ptr->type= UPDATE_TYPE_REQUERIES_PREFIX ; + if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) + ptr->type= UPDATE_TYPE_REQUIRES_PREFIX ; else ptr->type= UPDATE_TYPE; strmov(ptr->string, update_string.str); @@ -891,8 +917,10 @@ build_insert_string(void) dynstr_append_mem(&insert_string, ")", 1); - ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)); - ptr->string= (char *)my_malloc(insert_string.length+1, MYF(MY_WME)); + ptr= (statement *)my_malloc(sizeof(statement), + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); + ptr->string= (char *)my_malloc(insert_string.length + 1, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); ptr->length= insert_string.length+1; ptr->type= INSERT_TYPE; strmov(ptr->string, insert_string.str); @@ -902,19 +930,19 @@ build_insert_string(void) /* - build_query_string() + build_select_string() This function builds a query if the user opts to not supply a query statement or file containing a query statement */ static statement * -build_query_string(void) +build_select_string(my_bool key) { char buf[HUGE_STRING_LENGTH]; int col_count; statement *ptr; static DYNAMIC_STRING query_string; - DBUG_ENTER("build_query_string"); + DBUG_ENTER("build_select_string"); init_dynamic_string(&query_string, "", 1024, 1024); @@ -947,11 +975,22 @@ build_query_string(void) dynstr_append_mem(&query_string, ",", 1); } - dynstr_append_mem(&query_string, " FROM t1", 8); - ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)); - ptr->string= (char *)my_malloc(query_string.length+1, MYF(MY_WME)); + dynstr_append(&query_string, " FROM t1"); + + if ((key) && + (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary)) + dynstr_append(&query_string, " WHERE id = "); + + ptr= (statement *)my_malloc(sizeof(statement), + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); + ptr->string= (char *)my_malloc(query_string.length + 1, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); ptr->length= query_string.length+1; - ptr->type= SELECT_TYPE; + if ((key) && + (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary)) + ptr->type= SELECT_TYPE_REQUIRES_PREFIX; + else + ptr->type= SELECT_TYPE; strmov(ptr->string, query_string.str); dynstr_free(&query_string); DBUG_RETURN(ptr); @@ -986,7 +1025,8 @@ get_options(int *argc,char ***argv) exit(1); } - if (auto_generate_sql && auto_generate_sql_guid_primary && auto_generate_sql_autoincrement) + if (auto_generate_sql && auto_generate_sql_guid_primary && + auto_generate_sql_autoincrement) { fprintf(stderr, "%s: Either auto-generate-sql-guid-primary or auto-generate-sql-add-autoincrement can be used!\n", @@ -994,6 +1034,27 @@ get_options(int *argc,char ***argv) exit(1); } + if (auto_generate_sql && + ((auto_generate_sql_autoincrement == FALSE) || + (auto_generate_sql_guid_primary == FALSE)) && + auto_generate_sql_type == 'k') + { + fprintf(stderr, + "%s: Can't perform key test without a primary key!\n", + my_progname); + exit(1); + } + + + + if (auto_generate_sql && num_of_query && auto_actual_queries) + { + fprintf(stderr, + "%s: Either auto-generate-sql-execute-number or number-of-queries can be used!\n", + my_progname); + exit(1); + } + parse_comma(concurrency_str ? concurrency_str : "1", &concurrency); if (lock_directory) @@ -1029,18 +1090,48 @@ get_options(int *argc,char ***argv) unsigned long long x= 0; statement *ptr_statement; + if (verbose >= 2) + printf("Building Create Statements for Auto\n"); + create_statements= build_table_string(); + /* + Pre-populate table + */ + for (ptr_statement= create_statements, x= 0; + x < auto_generate_sql_unique_write_number; + x++, ptr_statement= ptr_statement->next) + { + ptr_statement->next= build_insert_string(); + } + + if (verbose >= 2) + printf("Building Query Statements for Auto\n"); if (auto_generate_sql_type[0] == 'r') { - for (ptr_statement= create_statements, x= 0; - x < auto_generate_sql_number; + if (verbose >= 2) + printf("Generating SELECT Statements for Auto\n"); + + query_statements= build_select_string(FALSE); + for (ptr_statement= query_statements, x= 0; + x < auto_generate_sql_unique_query_number; x++, ptr_statement= ptr_statement->next) { - ptr_statement->next= build_insert_string(); + ptr_statement->next= build_select_string(FALSE); } + } + else if (auto_generate_sql_type[0] == 'k') + { + if (verbose >= 2) + printf("Generating SELECT for keys Statements for Auto\n"); - query_statements= build_query_string(); + query_statements= build_select_string(TRUE); + for (ptr_statement= query_statements, x= 0; + x < auto_generate_sql_unique_query_number; + x++, ptr_statement= ptr_statement->next) + { + ptr_statement->next= build_select_string(TRUE); + } } else if (auto_generate_sql_type[0] == 'w') { @@ -1049,9 +1140,11 @@ get_options(int *argc,char ***argv) Archive (since strings which were identical one after another would be too easily optimized). */ + if (verbose >= 2) + printf("Generating INSERT Statements for Auto\n"); query_statements= build_insert_string(); for (ptr_statement= query_statements, x= 0; - x < auto_generate_sql_number; + x < auto_generate_sql_unique_query_number; x++, ptr_statement= ptr_statement->next) { ptr_statement->next= build_insert_string(); @@ -1059,16 +1152,9 @@ get_options(int *argc,char ***argv) } else if (auto_generate_sql_type[0] == 'u') { - for (ptr_statement= create_statements, x= 0; - x < auto_generate_sql_number; - x++, ptr_statement= ptr_statement->next) - { - ptr_statement->next= build_insert_string(); - } - query_statements= build_update_string(); for (ptr_statement= query_statements, x= 0; - x < auto_generate_sql_number; + x < auto_generate_sql_unique_query_number; x++, ptr_statement= ptr_statement->next) { ptr_statement->next= build_update_string(); @@ -1084,7 +1170,7 @@ get_options(int *argc,char ***argv) at the moment it results in "every other". */ for (ptr_statement= query_statements, x= 0; - x < 4; + x < auto_generate_sql_unique_query_number; x++, ptr_statement= ptr_statement->next) { if (coin) @@ -1094,7 +1180,7 @@ get_options(int *argc,char ***argv) } else { - ptr_statement->next= build_query_string(); + ptr_statement->next= build_select_string(TRUE); coin= 1; } } @@ -1116,12 +1202,13 @@ get_options(int *argc,char ***argv) fprintf(stderr,"%s: Could not open create file\n", my_progname); exit(1); } - tmp_string= (char *)my_malloc(sbuf.st_size+1, MYF(MY_WME)); + tmp_string= (char *)my_malloc(sbuf.st_size + 1, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); my_read(data_file, tmp_string, sbuf.st_size, MYF(0)); tmp_string[sbuf.st_size]= '\0'; my_close(data_file,MYF(0)); parse_delimiter(tmp_string, &create_statements, delimiter[0]); - my_free(tmp_string, MYF(0)); + my_free((gptr)tmp_string, MYF(0)); } else if (create_string) { @@ -1142,14 +1229,15 @@ get_options(int *argc,char ***argv) fprintf(stderr,"%s: Could not open query supplied file\n", my_progname); exit(1); } - tmp_string= (char *)my_malloc(sbuf.st_size+1, MYF(MY_WME)); + tmp_string= (char *)my_malloc(sbuf.st_size + 1, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); my_read(data_file, tmp_string, sbuf.st_size, MYF(0)); tmp_string[sbuf.st_size]= '\0'; my_close(data_file,MYF(0)); if (user_supplied_query) actual_queries= parse_delimiter(tmp_string, &query_statements, delimiter[0]); - my_free(tmp_string, MYF(0)); + my_free((gptr)tmp_string, MYF(0)); } else if (user_supplied_query) { @@ -1158,6 +1246,9 @@ get_options(int *argc,char ***argv) } } + if (verbose >= 2) + printf("Parsing engines to use.\n"); + if (default_engine) parse_delimiter(default_engine, &engine_statements, ','); @@ -1175,55 +1266,65 @@ static int run_query(MYSQL *mysql, const char *query, int len) return 0; } - if (verbose >= 2) + if (verbose >= 3) printf("%.*s;\n", len, query); return mysql_real_query(mysql, query, len); } static int -generate_primary_key_list(MYSQL *mysql) +generate_primary_key_list(MYSQL *mysql, statement *engine_stmt) { char query[HUGE_STRING_LENGTH]; int len; MYSQL_RES *result; MYSQL_ROW row; - char *ptr; + unsigned long long counter; DBUG_ENTER("create_schema"); - len= snprintf(query, HUGE_STRING_LENGTH, "SELECT id from t1"); - - if (run_query(mysql, query, len)) - { - fprintf(stderr,"%s: Cannot select GUID primary keys. (%s)\n", my_progname, - mysql_error(mysql)); - exit(1); - } - - if (opt_only_print) + /* + Blackhole is a special case, this allows us to test the upper end + of the server during load runs. + */ + if (opt_only_print || (engine_stmt && + strstr(engine_stmt->string, "blackhole"))) { primary_keys_number_of= 1; - primary_keys= (char *)my_malloc(sizeof(char) * primary_keys_number_of * 32, - MYF(MY_ZEROFILL)); - memcpy(primary_keys, "796c4422-1d94-102a-9d6d-00e0812d", 32); - DBUG_RETURN(0); + primary_keys= (char **)my_malloc(sizeof(char *) * primary_keys_number_of, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); + /* Yes, we strdup a const string to simplify the interface */ + primary_keys[0]= my_strdup("796c4422-1d94-102a-9d6d-00e0812d", MYF(0)); } - result= mysql_store_result(mysql); - DBUG_ASSERT(result); - primary_keys_number_of= mysql_num_rows(result); - /* - We know the GUID are 32 characters in length, so we preallocate. - */ - primary_keys= (char *)my_malloc(sizeof(char) * primary_keys_number_of * 32, - MYF(MY_ZEROFILL)); - ptr= primary_keys; - while ((row = mysql_fetch_row(result))) + else { - memcpy(ptr, row[0], 32); - ptr+= 32; + len= snprintf(query, HUGE_STRING_LENGTH, "SELECT id from t1"); + + if (run_query(mysql, query, len)) + { + fprintf(stderr,"%s: Cannot select GUID primary keys. (%s)\n", my_progname, + mysql_error(mysql)); + exit(1); + } + + result= mysql_store_result(mysql); + primary_keys_number_of= mysql_num_rows(result); + + /* So why check this? Blackhole :) */ + if (primary_keys_number_of) + { + /* + We create the structure and loop and create the items. + */ + primary_keys= (char **)my_malloc(sizeof(char *) * primary_keys_number_of, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); + row= mysql_fetch_row(result); + for (counter= 0; counter < primary_keys_number_of; + counter++, row= mysql_fetch_row(result)) + primary_keys[counter]= my_strdup(row[0], MYF(0)); + } + + mysql_free_result(result); } - DBUG_ASSERT(ptr == primary_keys + (primary_keys_number_of * 32)); - mysql_free_result(result); DBUG_RETURN(0); } @@ -1231,7 +1332,16 @@ generate_primary_key_list(MYSQL *mysql) static int drop_primary_key_list(void) { - my_free(primary_keys, MYF(0)); + unsigned long long counter; + + if (primary_keys_number_of) + { + for (counter= 0; counter < primary_keys_number_of; counter++) + my_free((gptr)primary_keys[counter], MYF(0)); + + my_free((gptr)primary_keys, MYF(0)); + } + return 0; } @@ -1241,11 +1351,16 @@ create_schema(MYSQL *mysql, const char *db, statement *stmt, { char query[HUGE_STRING_LENGTH]; statement *ptr; + statement *after_create; int len; + ulonglong count; DBUG_ENTER("create_schema"); len= snprintf(query, HUGE_STRING_LENGTH, "CREATE SCHEMA `%s`", db); + if (verbose >= 2) + printf("Loading Pre-data\n"); + if (run_query(mysql, query, len)) { fprintf(stderr,"%s: Cannot create schema %s : %s\n", my_progname, db, @@ -1259,8 +1374,9 @@ create_schema(MYSQL *mysql, const char *db, statement *stmt, } else { - if (verbose >= 2) + if (verbose >= 3) printf("%s;\n", query); + if (mysql_select_db(mysql, db)) { fprintf(stderr,"%s: Cannot select schema '%s': %s\n",my_progname, db, @@ -1281,8 +1397,15 @@ create_schema(MYSQL *mysql, const char *db, statement *stmt, } } - for (ptr= stmt; ptr && ptr->length; ptr= ptr->next) + count= 0; + after_create= stmt; + +limit_not_met: + for (ptr= after_create; ptr && ptr->length; ptr= ptr->next, count++) { + if (auto_generate_sql && ( auto_generate_sql_number == count)) + break; + if (run_query(mysql, ptr->string, ptr->length)) { fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", @@ -1291,6 +1414,13 @@ create_schema(MYSQL *mysql, const char *db, statement *stmt, } } + if (auto_generate_sql && (auto_generate_sql_number > count )) + { + /* Special case for auto create, we don't want to create tables twice */ + after_create= stmt->next; + goto limit_not_met; + } + DBUG_RETURN(0); } @@ -1379,10 +1509,9 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) /* child */ DBUG_PRINT("info", ("fork returned 0, calling task(\"%s\"), pid %d gid %d", stmts ? stmts->string : "", pid, getgid())); - if (verbose >= 2) - fprintf(stderr, - "%s: fork returned 0, calling task pid %d gid %d\n", - my_progname, pid, getgid()); + if (verbose >= 3) + printf("%s: fork returned 0, calling task pid %d gid %d\n", + my_progname, pid, getgid()); run_task(&con); exit(0); break; @@ -1398,9 +1527,9 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) default: /* parent, forked */ DBUG_PRINT("info", ("default, break: pid %d gid %d", pid, getgid())); - if (verbose >= 2) - fprintf(stderr,"%s: fork returned %d, gid %d\n", - my_progname, pid, getgid()); + if (verbose >= 3) + printf("%s: fork returned %d, gid %d\n", + my_progname, pid, getgid()); break; } } @@ -1437,8 +1566,11 @@ WAIT: pid= wait(&status); DBUG_PRINT("info", ("Parent: child %d status %d", pid, status)); if (status != 0) + { printf("%s: Child %d died with the status %d\n", my_progname, pid, status); + exit(0); + } } } #endif @@ -1468,10 +1600,18 @@ run_task(thread_context *con) DBUG_PRINT("info", ("task script \"%s\"", con->stmt ? con->stmt->string : "")); if (!(mysql= mysql_init(NULL))) - goto end; + { + fprintf(stderr,"%s: mysql_init() failed ERROR : %s\n", + my_progname, mysql_error(mysql)); + exit(0); + } if (con->thread && mysql_thread_init()) - goto end; + { + fprintf(stderr,"%s: mysql_thread_init() failed ERROR : %s\n", + my_progname, mysql_error(mysql)); + exit(0); + } DBUG_PRINT("info", ("trying to connect to host %s as user %s", host, user)); lock_file= my_open(lock_file_str, O_RDWR, MYF(0)); @@ -1504,7 +1644,7 @@ run_task(thread_context *con) } DBUG_PRINT("info", ("connected.")); if (verbose >= 3) - fprintf(stderr, "connected!\n"); + printf("connected!\n"); queries= 0; limit_not_met: @@ -1513,20 +1653,38 @@ limit_not_met: /* We have to execute differently based on query type. This should become a function. */ - if (ptr->type == UPDATE_TYPE_REQUERIES_PREFIX) + if ((ptr->type == UPDATE_TYPE_REQUIRES_PREFIX) || + (ptr->type == SELECT_TYPE_REQUIRES_PREFIX)) { - char buffer[HUGE_STRING_LENGTH]; - char *key= primary_keys + (random() % primary_keys_number_of); int length; + long int key_val; + char *key; + char buffer[HUGE_STRING_LENGTH]; - length= snprintf(buffer, HUGE_STRING_LENGTH, "%.*s '%.*s'", - (int)ptr->length, ptr->string, 32, key); - - if (run_query(mysql, buffer, length)) + /* + This should only happen if some sort of new engine was + implemented that didn't properly handle UPDATEs. + + Just in case someone runs this under an experimental engine we don't + want a crash so the if() is placed here. + */ + DBUG_ASSERT(primary_keys_number_of); + if (primary_keys_number_of) { - fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", - my_progname, (uint)length, buffer, mysql_error(mysql)); - goto end; + key_val= random() % primary_keys_number_of; + key= primary_keys[key_val]; + + DBUG_ASSERT(key); + + length= snprintf(buffer, HUGE_STRING_LENGTH, "%.*s '%s'", + (int)ptr->length, ptr->string, key); + + if (run_query(mysql, buffer, length)) + { + fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", + my_progname, (uint)length, buffer, mysql_error(mysql)); + exit(0); + } } } else @@ -1535,9 +1693,10 @@ limit_not_met: { fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", my_progname, (uint)ptr->length, ptr->string, mysql_error(mysql)); - goto end; + exit(0); } } + if (mysql_field_count(mysql)) { result= mysql_store_result(mysql); @@ -1581,9 +1740,11 @@ parse_delimiter(const char *script, statement **stmt, char delm) uint length= strlen(script); uint count= 0; /* We know that there is always one */ - for (tmp= *sptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)); + for (tmp= *sptr= (statement *)my_malloc(sizeof(statement), + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); (retstr= strchr(ptr, delm)); - tmp->next= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)), + tmp->next= (statement *)my_malloc(sizeof(statement), + MYF(MY_ZEROFILL|MY_FAE|MY_WME)), tmp= tmp->next) { count++; @@ -1619,7 +1780,8 @@ parse_comma(const char *string, uint **range) if (*ptr == ',') count++; /* One extra spot for the NULL */ - nptr= *range= (uint *)my_malloc(sizeof(uint) * (count + 1), MYF(MY_ZEROFILL)); + nptr= *range= (uint *)my_malloc(sizeof(uint) * (count + 1), + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); ptr= (char *)string; x= 0; @@ -1670,7 +1832,7 @@ void generate_stats(conclusions *con, statement *eng, stats *sptr) { stats *ptr; - int x; + unsigned int x; con->min_timing= sptr->timing; con->max_timing= sptr->timing; @@ -1710,7 +1872,7 @@ statement_cleanup(statement *stmt) { nptr= ptr->next; if (ptr->string) - my_free(ptr->string, MYF(0)); - my_free((byte *)ptr, MYF(0)); + my_free((gptr)ptr->string, MYF(0)); + my_free((gptr)(byte *)ptr, MYF(0)); } } diff --git a/mysql-test/t/mysqlslap.test b/mysql-test/t/mysqlslap.test index c98da4b4a0f..5d98dac03ea 100644 --- a/mysql-test/t/mysqlslap.test +++ b/mysql-test/t/mysqlslap.test @@ -28,3 +28,7 @@ --exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-add-autoincrement --auto-generate-sql-load-type=mixed --exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --auto-generate-sql-load-type=update + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --auto-generate-sql-load-type=update --auto-generate-sql-execute-number=5 + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --auto-generate-sql-load-type=key --auto-generate-sql-execute-number=5 From 70fa527d9e6fe32bf3432a9e054860aac1f12666 Mon Sep 17 00:00:00 2001 From: "brian@zim.(none)" <> Date: Tue, 13 Mar 2007 00:56:30 -0700 Subject: [PATCH 03/28] Cleaning up 4 pushbuild warnings caught by Windows. --- client/mysqlslap.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/client/mysqlslap.c b/client/mysqlslap.c index 1ec8d124a48..1976c54c393 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -1037,7 +1037,7 @@ get_options(int *argc,char ***argv) if (auto_generate_sql && ((auto_generate_sql_autoincrement == FALSE) || (auto_generate_sql_guid_primary == FALSE)) && - auto_generate_sql_type == 'k') + auto_generate_sql_type[0] == 'k') { fprintf(stderr, "%s: Can't perform key test without a primary key!\n", @@ -1290,7 +1290,8 @@ generate_primary_key_list(MYSQL *mysql, statement *engine_stmt) strstr(engine_stmt->string, "blackhole"))) { primary_keys_number_of= 1; - primary_keys= (char **)my_malloc(sizeof(char *) * primary_keys_number_of, + primary_keys= (char **)my_malloc((uint)(sizeof(char *) * + primary_keys_number_of), MYF(MY_ZEROFILL|MY_FAE|MY_WME)); /* Yes, we strdup a const string to simplify the interface */ primary_keys[0]= my_strdup("796c4422-1d94-102a-9d6d-00e0812d", MYF(0)); @@ -1315,7 +1316,8 @@ generate_primary_key_list(MYSQL *mysql, statement *engine_stmt) /* We create the structure and loop and create the items. */ - primary_keys= (char **)my_malloc(sizeof(char *) * primary_keys_number_of, + primary_keys= (char **)my_malloc((uint)(sizeof(char *) * + primary_keys_number_of), MYF(MY_ZEROFILL|MY_FAE|MY_WME)); row= mysql_fetch_row(result); for (counter= 0; counter < primary_keys_number_of; @@ -1657,7 +1659,7 @@ limit_not_met: (ptr->type == SELECT_TYPE_REQUIRES_PREFIX)) { int length; - long int key_val; + unsigned int key_val; char *key; char buffer[HUGE_STRING_LENGTH]; @@ -1671,7 +1673,7 @@ limit_not_met: DBUG_ASSERT(primary_keys_number_of); if (primary_keys_number_of) { - key_val= random() % primary_keys_number_of; + key_val= (unsigned int)(random() % primary_keys_number_of); key= primary_keys[key_val]; DBUG_ASSERT(key); From deef4ef274433afbfa4ea0c7464b956faa9956e6 Mon Sep 17 00:00:00 2001 From: "brian@zim.(none)" <> Date: Tue, 13 Mar 2007 10:27:59 -0700 Subject: [PATCH 04/28] Added comment, and fixed test. --- client/mysqlslap.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/client/mysqlslap.c b/client/mysqlslap.c index 1976c54c393..1dc49aa06f3 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -1034,16 +1034,19 @@ get_options(int *argc,char ***argv) exit(1); } - if (auto_generate_sql && - ((auto_generate_sql_autoincrement == FALSE) || - (auto_generate_sql_guid_primary == FALSE)) && - auto_generate_sql_type[0] == 'k') - { + /* + We are testing to make sure that if someone specified a key search + that we actually added a key! + */ + if (auto_generate_sql && auto_generate_sql_type[0] == 'k') + if ( auto_generate_sql_autoincrement == FALSE && + auto_generate_sql_guid_primary == FALSE) + { fprintf(stderr, "%s: Can't perform key test without a primary key!\n", my_progname); exit(1); - } + } From 45d7dff67d850461cde4cb83846740d6179d351c Mon Sep 17 00:00:00 2001 From: "brian@zim.(none)" <> Date: Thu, 15 Mar 2007 23:39:07 -0700 Subject: [PATCH 05/28] Correctly report load type. Updated engine to also handle create options Secondary indexes can now be generated (aka the test PeterZ thoughts!) --- client/client_priv.h | 1 + client/mysqlslap.c | 331 +++++++++++++++++++++++++++++------- mysql-test/t/mysqlslap.test | 2 + sql/authors.h | 1 + 4 files changed, 269 insertions(+), 66 deletions(-) diff --git a/client/client_priv.h b/client/client_priv.h index 00ef1a79d96..b6543870528 100644 --- a/client/client_priv.h +++ b/client/client_priv.h @@ -61,6 +61,7 @@ enum options_client OPT_SLAP_AUTO_GENERATE_ADD_AUTO, OPT_SLAP_AUTO_GENERATE_GUID_PRIMARY, OPT_SLAP_AUTO_GENERATE_EXECUTE_QUERIES, + OPT_SLAP_AUTO_GENERATE_SECONDARY_INDEXES, OPT_SLAP_AUTO_GENERATE_UNIQUE_WRITE_NUM, OPT_SLAP_AUTO_GENERATE_UNIQUE_QUERY_NUM, OPT_MYSQL_REPLACE_INTO, OPT_BASE64_OUTPUT, OPT_SERVER_ID, diff --git a/client/mysqlslap.c b/client/mysqlslap.c index 1dc49aa06f3..0010d8e46ae 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -145,13 +145,22 @@ const char *auto_generate_sql_type= "mixed"; static unsigned long connect_flags= CLIENT_MULTI_RESULTS; -static int verbose, num_int_cols, num_char_cols, delimiter_length; +static int verbose, delimiter_length; +const char *num_int_cols_opt; +const char *num_char_cols_opt; +/* Yes, we do set defaults here */ +static unsigned int num_int_cols= 1; +static unsigned int num_char_cols= 1; +static unsigned int num_int_cols_index= 0; +static unsigned int num_char_cols_index= 0; + static unsigned int iterations; static char *default_charset= (char*) MYSQL_DEFAULT_CHARSET_NAME; static ulonglong actual_queries= 0; static ulonglong auto_actual_queries; static ulonglong auto_generate_sql_unique_write_number; static ulonglong auto_generate_sql_unique_query_number; +static unsigned int auto_generate_sql_secondary_indexes; static ulonglong num_of_query; static ulonglong auto_generate_sql_number; const char *concurrency_str= NULL; @@ -176,9 +185,21 @@ struct statement { char *string; size_t length; unsigned char type; + char *option; + size_t option_length; statement *next; }; +typedef struct option_string option_string; + +struct option_string { + char *string; + size_t length; + char *option; + size_t option_length; + option_string *next; +}; + typedef struct stats stats; struct stats { @@ -209,30 +230,32 @@ struct conclusions { unsigned long long min_rows; }; +static option_string *engine_options= NULL; static statement *create_statements= NULL, - *engine_statements= NULL, *query_statements= NULL; /* Prototypes */ void print_conclusions(conclusions *con); void print_conclusions_csv(conclusions *con); -void generate_stats(conclusions *con, statement *eng, stats *sptr); +void generate_stats(conclusions *con, option_string *eng, stats *sptr); uint parse_comma(const char *string, uint **range); uint parse_delimiter(const char *script, statement **stmt, char delm); +uint parse_option(const char *origin, option_string **stmt, char delm); static int drop_schema(MYSQL *mysql, const char *db); uint get_random_string(char *buf); static statement *build_table_string(void); static statement *build_insert_string(void); static statement *build_update_string(void); static statement * build_select_string(my_bool key); -static int generate_primary_key_list(MYSQL *mysql, statement *engine_stmt); +static int generate_primary_key_list(MYSQL *mysql, option_string *engine_stmt); static int drop_primary_key_list(void); static int create_schema(MYSQL *mysql, const char *db, statement *stmt, - statement *engine_stmt); + option_string *engine_stmt); static int run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit); int run_task(thread_context *con); void statement_cleanup(statement *stmt); +void option_cleanup(option_string *stmt); static const char ALPHANUMERICS[]= "0123456789ABCDEFGHIJKLMNOPQRSTWXYZabcdefghijklmnopqrstuvwxyz"; @@ -268,7 +291,7 @@ int main(int argc, char **argv) MYSQL mysql; unsigned int x; unsigned long long client_limit; - statement *eptr; + option_string *eptr; #ifdef __WIN__ opt_use_threads= 1; @@ -330,7 +353,7 @@ int main(int argc, char **argv) } /* Main iterations loop */ - eptr= engine_statements; + eptr= engine_options; do { /* For the final stage we run whatever queries we were asked to run */ @@ -419,8 +442,8 @@ int main(int argc, char **argv) my_free((gptr)concurrency, MYF(0)); statement_cleanup(create_statements); - statement_cleanup(engine_statements); statement_cleanup(query_statements); + option_cleanup(engine_options); #ifdef HAVE_SMEM if (shared_memory_base_name) @@ -456,9 +479,15 @@ static struct my_option my_long_options[] = (gptr*) &auto_generate_sql_guid_primary, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"auto-generate-sql-load-type", OPT_SLAP_AUTO_GENERATE_SQL_LOAD_TYPE, - "Load types are mixed, update, write, or read. Default is mixed\n", + "Load types are mixed, update, write, key, or read. Default is mixed\n", (gptr*) &auto_generate_sql_type, (gptr*) &auto_generate_sql_type, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"auto-generate-sql-secondary-indexes", + OPT_SLAP_AUTO_GENERATE_SECONDARY_INDEXES, + "Number of secondary indexes to add auto-generated tables.", + (gptr*) &auto_generate_sql_secondary_indexes, + (gptr*) &auto_generate_sql_secondary_indexes, 0, + GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"auto-generate-sql-unique-query-number", OPT_SLAP_AUTO_GENERATE_UNIQUE_QUERY_NUM, "Number of unique queries auto tests", @@ -510,12 +539,12 @@ static struct my_option my_long_options[] = REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"number-char-cols", 'x', "Number of VARCHAR columns to create table with if specifying --auto-generate-sql ", - (gptr*) &num_char_cols, (gptr*) &num_char_cols, 0, GET_UINT, REQUIRED_ARG, - 1, 0, 0, 0, 0, 0}, + (gptr*) &num_char_cols_opt, (gptr*) &num_char_cols_opt, 0, GET_STR, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, {"number-int-cols", 'y', "Number of INT columns to create table with if specifying --auto-generate-sql.", - (gptr*) &num_int_cols, (gptr*) &num_int_cols, 0, GET_UINT, REQUIRED_ARG, - 1, 0, 0, 0, 0, 0}, + (gptr*) &num_int_cols_opt, (gptr*) &num_int_cols_opt, 0, GET_STR, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, {"number-of-queries", OPT_MYSQL_NUMBER_OF_QUERY, "Limit each client to this number of queries (this is not exact).", (gptr*) &num_of_query, (gptr*) &num_of_query, 0, @@ -685,12 +714,12 @@ static statement * build_table_string(void) { char buf[HUGE_STRING_LENGTH]; - int col_count; + unsigned int col_count; statement *ptr; DYNAMIC_STRING table_string; DBUG_ENTER("build_table_string"); - DBUG_PRINT("info", ("num int cols %d num char cols %d", + DBUG_PRINT("info", ("num int cols %u num char cols %u", num_int_cols, num_char_cols)); init_dynamic_string(&table_string, "", 1024, 1024); @@ -699,12 +728,7 @@ build_table_string(void) if (auto_generate_sql_autoincrement) { - if (snprintf(buf, HUGE_STRING_LENGTH, "id serial") > HUGE_STRING_LENGTH) - { - fprintf(stderr, "Memory Allocation error in create table\n"); - exit(1); - } - dynstr_append(&table_string, buf); + dynstr_append(&table_string, "id serial"); if (num_int_cols || num_char_cols) dynstr_append(&table_string, ","); @@ -712,12 +736,29 @@ build_table_string(void) if (auto_generate_sql_guid_primary) { - if (snprintf(buf, HUGE_STRING_LENGTH, "id varchar(32) primary key") > HUGE_STRING_LENGTH) + dynstr_append(&table_string, "id varchar(32) primary key"); + + if (num_int_cols || num_char_cols || auto_generate_sql_guid_primary) + dynstr_append(&table_string, ","); + } + + if (auto_generate_sql_secondary_indexes) + { + unsigned int count; + + for (count= 0; count < auto_generate_sql_secondary_indexes; count++) { + if (count) /* Except for the first pass we add a comma */ + dynstr_append(&table_string, ","); + + if (snprintf(buf, HUGE_STRING_LENGTH, "id%d varchar(32) unique key", count) + > HUGE_STRING_LENGTH) + { fprintf(stderr, "Memory Allocation error in create table\n"); exit(1); + } + dynstr_append(&table_string, buf); } - dynstr_append(&table_string, buf); if (num_int_cols || num_char_cols) dynstr_append(&table_string, ","); @@ -726,11 +767,23 @@ build_table_string(void) if (num_int_cols) for (col_count= 1; col_count <= num_int_cols; col_count++) { - if (snprintf(buf, HUGE_STRING_LENGTH, "intcol%d INT(32)", col_count) - > HUGE_STRING_LENGTH) + if (num_int_cols_index) { - fprintf(stderr, "Memory Allocation error in create table\n"); - exit(1); + if (snprintf(buf, HUGE_STRING_LENGTH, "intcol%d INT(32), INDEX(intcol%d)", + col_count, col_count) > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in create table\n"); + exit(1); + } + } + else + { + if (snprintf(buf, HUGE_STRING_LENGTH, "intcol%d INT(32) ", col_count) + > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in create table\n"); + exit(1); + } } dynstr_append(&table_string, buf); @@ -741,11 +794,24 @@ build_table_string(void) if (num_char_cols) for (col_count= 1; col_count <= num_char_cols; col_count++) { - if (snprintf(buf, HUGE_STRING_LENGTH, "charcol%d VARCHAR(128)", col_count) - > HUGE_STRING_LENGTH) + if (num_char_cols_index) { - fprintf(stderr, "Memory Allocation error in creating table\n"); - exit(1); + if (snprintf(buf, HUGE_STRING_LENGTH, + "charcol%d VARCHAR(128), INDEX(charcol%d) ", + col_count, col_count) > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating table\n"); + exit(1); + } + } + else + { + if (snprintf(buf, HUGE_STRING_LENGTH, "charcol%d VARCHAR(128)", + col_count) > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating table\n"); + exit(1); + } } dynstr_append(&table_string, buf); @@ -759,6 +825,7 @@ build_table_string(void) ptr->string = (char *)my_malloc(table_string.length+1, MYF(MY_ZEROFILL|MY_FAE|MY_WME)); ptr->length= table_string.length+1; + ptr->type= CREATE_TABLE_TYPE; strmov(ptr->string, table_string.str); dynstr_free(&table_string); DBUG_RETURN(ptr); @@ -774,7 +841,7 @@ static statement * build_update_string(void) { char buf[HUGE_STRING_LENGTH]; - int col_count; + unsigned int col_count; statement *ptr; DYNAMIC_STRING update_string; DBUG_ENTER("build_update_string"); @@ -818,14 +885,7 @@ build_update_string(void) } if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) - { - if (snprintf(buf, HUGE_STRING_LENGTH, " WHERE id = ") > HUGE_STRING_LENGTH) - { - fprintf(stderr, "Memory Allocation error in creating update_string\n"); - exit(1); - } - dynstr_append(&update_string, buf); - } + dynstr_append(&update_string, " WHERE id = "); ptr= (statement *)my_malloc(sizeof(statement), @@ -854,7 +914,7 @@ static statement * build_insert_string(void) { char buf[HUGE_STRING_LENGTH]; - int col_count; + unsigned int col_count; statement *ptr; DYNAMIC_STRING insert_string; DBUG_ENTER("build_insert_string"); @@ -865,12 +925,7 @@ build_insert_string(void) if (auto_generate_sql_autoincrement) { - if (snprintf(buf, HUGE_STRING_LENGTH, "NULL") > HUGE_STRING_LENGTH) - { - fprintf(stderr, "Memory Allocation error in creating insert\n"); - exit(1); - } - dynstr_append(&insert_string, buf); + dynstr_append(&insert_string, "NULL"); if (num_int_cols || num_char_cols) dynstr_append(&insert_string, ","); @@ -878,12 +933,23 @@ build_insert_string(void) if (auto_generate_sql_guid_primary) { - if (snprintf(buf, HUGE_STRING_LENGTH, "uuid()") > HUGE_STRING_LENGTH) + dynstr_append(&insert_string, "uuid()"); + + if (num_int_cols || num_char_cols) + dynstr_append(&insert_string, ","); + } + + if (auto_generate_sql_secondary_indexes) + { + unsigned int count; + + for (count= 0; count < auto_generate_sql_secondary_indexes; count++) { - fprintf(stderr, "Memory Allocation error in create table\n"); - exit(1); + if (count) /* Except for the first pass we add a comma */ + dynstr_append(&insert_string, ","); + + dynstr_append(&insert_string, "uuid()"); } - dynstr_append(&insert_string, buf); if (num_int_cols || num_char_cols) dynstr_append(&insert_string, ","); @@ -939,7 +1005,7 @@ static statement * build_select_string(my_bool key) { char buf[HUGE_STRING_LENGTH]; - int col_count; + unsigned int col_count; statement *ptr; static DYNAMIC_STRING query_string; DBUG_ENTER("build_select_string"); @@ -1088,6 +1154,29 @@ get_options(int *argc,char ***argv) if (opt_only_print) opt_silent= TRUE; + if (num_int_cols_opt) + { + option_string *str; + parse_option(num_int_cols_opt, &str, ','); + num_int_cols= atoi(str->string); + if (str->option) + num_int_cols_index= atoi(str->option); + option_cleanup(str); + } + + if (num_char_cols_opt) + { + option_string *str; + parse_option(num_char_cols_opt, &str, ','); + num_char_cols= atoi(str->string); + if (str->option) + num_char_cols_index= atoi(str->option); + else + num_char_cols_index= 0; + option_cleanup(str); + } + + if (auto_generate_sql) { unsigned long long x= 0; @@ -1253,7 +1342,7 @@ get_options(int *argc,char ***argv) printf("Parsing engines to use.\n"); if (default_engine) - parse_delimiter(default_engine, &engine_statements, ','); + parse_option(default_engine, &engine_options, ','); if (tty_password) opt_password= get_tty_password(NullS); @@ -1276,10 +1365,8 @@ static int run_query(MYSQL *mysql, const char *query, int len) static int -generate_primary_key_list(MYSQL *mysql, statement *engine_stmt) +generate_primary_key_list(MYSQL *mysql, option_string *engine_stmt) { - char query[HUGE_STRING_LENGTH]; - int len; MYSQL_RES *result; MYSQL_ROW row; unsigned long long counter; @@ -1301,9 +1388,7 @@ generate_primary_key_list(MYSQL *mysql, statement *engine_stmt) } else { - len= snprintf(query, HUGE_STRING_LENGTH, "SELECT id from t1"); - - if (run_query(mysql, query, len)) + if (run_query(mysql, "SELECT id from t1", strlen("SELECT id from t1"))) { fprintf(stderr,"%s: Cannot select GUID primary keys. (%s)\n", my_progname, mysql_error(mysql)); @@ -1352,7 +1437,7 @@ drop_primary_key_list(void) static int create_schema(MYSQL *mysql, const char *db, statement *stmt, - statement *engine_stmt) + option_string *engine_stmt) { char query[HUGE_STRING_LENGTH]; statement *ptr; @@ -1411,11 +1496,27 @@ limit_not_met: if (auto_generate_sql && ( auto_generate_sql_number == count)) break; - if (run_query(mysql, ptr->string, ptr->length)) + if (engine_stmt && engine_stmt->option && ptr->type == CREATE_TABLE_TYPE) { - fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", - my_progname, (uint)ptr->length, ptr->string, mysql_error(mysql)); - exit(1); + char buffer[HUGE_STRING_LENGTH]; + + snprintf(buffer, HUGE_STRING_LENGTH, "%s %s", ptr->string, + engine_stmt->option); + if (run_query(mysql, buffer, strlen(buffer))) + { + fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", + my_progname, (uint)ptr->length, ptr->string, mysql_error(mysql)); + exit(1); + } + } + else + { + if (run_query(mysql, ptr->string, ptr->length)) + { + fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", + my_progname, (uint)ptr->length, ptr->string, mysql_error(mysql)); + exit(1); + } } } @@ -1734,6 +1835,84 @@ end: DBUG_RETURN(0); } +uint +parse_option(const char *origin, option_string **stmt, char delm) +{ + char *retstr; + char *ptr= (char *)origin; + option_string **sptr= stmt; + option_string *tmp; + uint length= strlen(origin); + uint count= 0; /* We know that there is always one */ + + for (tmp= *sptr= (option_string *)my_malloc(sizeof(option_string), + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); + (retstr= strchr(ptr, delm)); + tmp->next= (option_string *)my_malloc(sizeof(option_string), + MYF(MY_ZEROFILL|MY_FAE|MY_WME)), + tmp= tmp->next) + { + char buffer[HUGE_STRING_LENGTH]; + char *buffer_ptr; + + count++; + strncpy(buffer, ptr, (size_t)(retstr - ptr)); + if ((buffer_ptr= strchr(buffer, ':'))) + { + char *option_ptr; + + tmp->length= (size_t)(buffer_ptr - buffer); + tmp->string= my_strndup(ptr, tmp->length, MYF(MY_FAE)); + + option_ptr= ptr + 1 + tmp->length; + + /* Move past the : and the first string */ + tmp->option_length= (size_t)(retstr - option_ptr); + tmp->option= my_strndup(option_ptr, tmp->option_length, + MYF(MY_FAE)); + } + else + { + tmp->string= my_strndup(ptr, (size_t)(retstr - ptr), MYF(MY_FAE)); + tmp->length= (size_t)(retstr - ptr); + } + + ptr+= retstr - ptr + 1; + if (isspace(*ptr)) + ptr++; + count++; + } + + if (ptr != origin+length) + { + char *origin_ptr; + + if ((origin_ptr= strchr(ptr, ':'))) + { + char *option_ptr; + + tmp->length= (size_t)(origin_ptr - ptr); + tmp->string= my_strndup(origin, tmp->length, MYF(MY_FAE)); + + option_ptr= (char *)ptr + 1 + tmp->length; + + /* Move past the : and the first string */ + tmp->option_length= (size_t)((ptr + length) - option_ptr); + tmp->option= my_strndup(option_ptr, tmp->option_length, + MYF(MY_FAE)); + } + else + { + tmp->length= (size_t)((ptr + length) - ptr); + tmp->string= my_strndup(ptr, tmp->length, MYF(MY_FAE)); + } + + count++; + } + + return count; +} + uint parse_delimiter(const char *script, statement **stmt, char delm) @@ -1821,9 +2000,11 @@ void print_conclusions_csv(conclusions *con) { char buffer[HUGE_STRING_LENGTH]; + const char *ptr= auto_generate_sql_type ? auto_generate_sql_type : "query"; snprintf(buffer, HUGE_STRING_LENGTH, - "%s,query,%ld.%03ld,%ld.%03ld,%ld.%03ld,%d,%llu\n", + "%s,%s,%ld.%03ld,%ld.%03ld,%ld.%03ld,%d,%llu\n", con->engine ? con->engine : "", /* Storage engine we ran against */ + ptr, /* Load type */ con->avg_timing / 1000, con->avg_timing % 1000, /* Time to load */ con->min_timing / 1000, con->min_timing % 1000, /* Min time */ con->max_timing / 1000, con->max_timing % 1000, /* Max time */ @@ -1834,7 +2015,7 @@ print_conclusions_csv(conclusions *con) } void -generate_stats(conclusions *con, statement *eng, stats *sptr) +generate_stats(conclusions *con, option_string *eng, stats *sptr) { stats *ptr; unsigned int x; @@ -1866,6 +2047,24 @@ generate_stats(conclusions *con, statement *eng, stats *sptr) con->engine= NULL; } +void +option_cleanup(option_string *stmt) +{ + option_string *ptr, *nptr; + if (!stmt) + return; + + for (ptr= stmt; ptr; ptr= nptr) + { + nptr= ptr->next; + if (ptr->string) + my_free((gptr)ptr->string, MYF(0)); + if (ptr->option) + my_free((gptr)ptr->option, MYF(0)); + my_free((gptr)(byte *)ptr, MYF(0)); + } +} + void statement_cleanup(statement *stmt) { diff --git a/mysql-test/t/mysqlslap.test b/mysql-test/t/mysqlslap.test index 5d98dac03ea..f2bb9a72ecf 100644 --- a/mysql-test/t/mysqlslap.test +++ b/mysql-test/t/mysqlslap.test @@ -32,3 +32,5 @@ --exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --auto-generate-sql-load-type=update --auto-generate-sql-execute-number=5 --exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --auto-generate-sql-load-type=key --auto-generate-sql-execute-number=5 + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --auto-generate-sql-load-type=key --auto-generate-sql-execute-number=5 --auto-generate-sql-secondary-indexes=3 diff --git a/sql/authors.h b/sql/authors.h index 48b807c7884..dfe3b143e2f 100644 --- a/sql/authors.h +++ b/sql/authors.h @@ -66,6 +66,7 @@ struct show_table_authors_st show_table_authors[]= { "Parser, port to OS/2, storage engines and some random stuff" }, { "Yuri Dario", "", "OS/2 port" }, { "Andrei Elkin", "Espoo, Finland", "Replication" }, + { "Patrick Galbraith", "Sharon, NH", "Federated Engine, mysqlslap" }, { "Sergei Golubchik", "Kerpen, Germany", "Full-text search, precision math" }, { "Lenz Grimmer", "Hamburg, Germany", From e246cb3d80b1fa55480aa1204e547bb38b2ea02e Mon Sep 17 00:00:00 2001 From: "brian@zim.(none)" <> Date: Fri, 16 Mar 2007 15:20:22 -0700 Subject: [PATCH 06/28] The pthread() support was not working. Once I fixed use-thread in a previous push I realized that the pthread/windows code was not working. I've replaced the fork/thread design with a pure pthread design using condition timers and broadcast. Ramification, UNIX now uses thread, support for slaves had to be dropped and there is no need for the --use-threads flag. Added --concurrency=0 option so that it will start at 1 and keep going up until something bad happens :) --- client/client_priv.h | 1 - client/mysqlslap.c | 355 ++++++++++++++--------------------- mysql-test/mysql-test-run.pl | 3 +- mysql-test/t/mysqlslap.test | 2 +- 4 files changed, 144 insertions(+), 217 deletions(-) diff --git a/client/client_priv.h b/client/client_priv.h index b6543870528..6227dcdee44 100644 --- a/client/client_priv.h +++ b/client/client_priv.h @@ -49,7 +49,6 @@ enum options_client OPT_TRIGGERS, OPT_MYSQL_ONLY_PRINT, OPT_MYSQL_LOCK_DIRECTORY, - OPT_MYSQL_SLAP_SLAVE, OPT_USE_THREADS, OPT_IMPORT_USE_THREADS, OPT_MYSQL_NUMBER_OF_QUERY, diff --git a/client/mysqlslap.c b/client/mysqlslap.c index 0010d8e46ae..1b4692848b9 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -62,7 +62,6 @@ TODO: Add language for better tests String length for files and those put on the command line are not setup to handle binary data. - Report results of each thread into the lock file we use. More stats Break up tests and run them on multiple hosts at once. Allow output to be fed into a database directly. @@ -83,9 +82,7 @@ TODO: #define SELECT_TYPE_REQUIRES_PREFIX 5 #include "client_priv.h" -#ifdef HAVE_LIBPTHREAD #include -#endif #include #include #include @@ -100,9 +97,6 @@ TODO: #endif #include -#define MYSLAPLOCK "/myslaplock.lck" -#define MYSLAPLOCK_DIR "/tmp" - #ifdef __WIN__ #define srandom srand #define random rand @@ -113,6 +107,14 @@ TODO: static char *shared_memory_base_name=0; #endif +/* Global Thread counter */ +uint thread_counter; +pthread_mutex_t counter_mutex; +pthread_cond_t count_threshhold; +uint master_wakeup; +pthread_mutex_t sleeper_mutex; +pthread_cond_t sleep_threshhold; + static char **defaults_argv; char **primary_keys; @@ -127,15 +129,10 @@ const char *delimiter= "\n"; const char *create_schema_string= "mysqlslap"; -const char *lock_directory; -char lock_file_str[FN_REFLEN]; - static my_bool opt_preserve; static my_bool opt_only_print= FALSE; -static my_bool opt_slave; - static my_bool opt_compress= FALSE, tty_password= FALSE, opt_silent= FALSE, auto_generate_sql_autoincrement= FALSE, @@ -175,7 +172,6 @@ static uint opt_protocol= 0; static int get_options(int *argc,char ***argv); static uint opt_mysql_port= 0; -static my_bool opt_use_threads; static const char *load_default_groups[]= { "mysqlslap","client",0 }; @@ -213,7 +209,6 @@ typedef struct thread_context thread_context; struct thread_context { statement *stmt; ulonglong limit; - bool thread; }; typedef struct conclusions conclusions; @@ -256,6 +251,7 @@ static int run_scheduler(stats *sptr, statement *stmts, uint concur, int run_task(thread_context *con); void statement_cleanup(statement *stmt); void option_cleanup(option_string *stmt); +void concurrency_loop(MYSQL *mysql, uint current, option_string *eptr); static const char ALPHANUMERICS[]= "0123456789ABCDEFGHIJKLMNOPQRSTWXYZabcdefghijklmnopqrstuvwxyz"; @@ -289,14 +285,8 @@ static int gettimeofday(struct timeval *tp, void *tzp) int main(int argc, char **argv) { MYSQL mysql; - unsigned int x; - unsigned long long client_limit; option_string *eptr; -#ifdef __WIN__ - opt_use_threads= 1; -#endif - MY_INIT(argv[0]); load_defaults("my",load_default_groups,&argc,&argv); @@ -352,75 +342,33 @@ int main(int argc, char **argv) } } + VOID(pthread_mutex_init(&counter_mutex, NULL)); + VOID(pthread_cond_init(&count_threshhold, NULL)); + VOID(pthread_mutex_init(&sleeper_mutex, NULL)); + VOID(pthread_cond_init(&sleep_threshhold, NULL)); + /* Main iterations loop */ eptr= engine_options; do { /* For the final stage we run whatever queries we were asked to run */ uint *current; - conclusions conclusion; if (verbose >= 2) printf("Starting Concurrency Test\n"); - for (current= concurrency; current && *current; current++) + if (*concurrency) { - stats *head_sptr; - stats *sptr; - - head_sptr= (stats *)my_malloc(sizeof(stats) * iterations, - MYF(MY_ZEROFILL|MY_FAE|MY_WME)); - - bzero(&conclusion, sizeof(conclusions)); - - if (auto_actual_queries) - client_limit= auto_actual_queries; - else if (num_of_query) - client_limit= num_of_query / *current; - else - client_limit= actual_queries; - - for (x= 0, sptr= head_sptr; x < iterations; x++, sptr++) - { - /* - We might not want to load any data, such as when we are calling - a stored_procedure that doesn't use data, or we know we already have - data in the table. - */ - if (!opt_preserve) - drop_schema(&mysql, create_schema_string); - - /* First we create */ - if (create_statements) - create_schema(&mysql, create_schema_string, create_statements, eptr); - - /* - If we generated GUID we need to build a list of them from creation that - we can later use. - */ - if (verbose >= 2) - printf("Generating primary key list\n"); - if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) - generate_primary_key_list(&mysql, eptr); - - run_scheduler(sptr, query_statements, *current, client_limit); - - /* We are finished with this run */ - if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) - drop_primary_key_list(); + for (current= concurrency; current && *current; current++) + concurrency_loop(&mysql, *current, eptr); + } + else + { + uint infinite= 1; + do { + concurrency_loop(&mysql, infinite, eptr); } - - if (verbose >= 2) - printf("Generating stats\n"); - - generate_stats(&conclusion, eptr, head_sptr); - - if (!opt_silent) - print_conclusions(&conclusion); - if (opt_csv_str) - print_conclusions_csv(&conclusion); - - my_free((gptr)head_sptr, MYF(0)); + while (infinite++); } if (!opt_preserve) @@ -428,13 +376,14 @@ int main(int argc, char **argv) } while (eptr ? (eptr= eptr->next) : 0); + VOID(pthread_mutex_destroy(&counter_mutex)); + VOID(pthread_cond_destroy(&count_threshhold)); + VOID(pthread_mutex_destroy(&sleeper_mutex)); + VOID(pthread_cond_destroy(&sleep_threshhold)); + if (!opt_only_print) mysql_close(&mysql); /* Close & free connection */ - - /* Remove lock file */ - my_delete(lock_file_str, MYF(0)); - /* now free all the strings we created */ if (opt_password) my_free((gptr)opt_password, MYF(0)); @@ -455,6 +404,70 @@ int main(int argc, char **argv) return 0; } +void concurrency_loop(MYSQL *mysql, uint current, option_string *eptr) +{ + unsigned int x; + stats *head_sptr; + stats *sptr; + conclusions conclusion; + unsigned long long client_limit; + + head_sptr= (stats *)my_malloc(sizeof(stats) * iterations, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); + + bzero(&conclusion, sizeof(conclusions)); + + if (auto_actual_queries) + client_limit= auto_actual_queries; + else if (num_of_query) + client_limit= num_of_query / current; + else + client_limit= actual_queries; + + for (x= 0, sptr= head_sptr; x < iterations; x++, sptr++) + { + /* + We might not want to load any data, such as when we are calling + a stored_procedure that doesn't use data, or we know we already have + data in the table. + */ + if (!opt_preserve) + drop_schema(mysql, create_schema_string); + + /* First we create */ + if (create_statements) + create_schema(mysql, create_schema_string, create_statements, eptr); + + /* + If we generated GUID we need to build a list of them from creation that + we can later use. + */ + if (verbose >= 2) + printf("Generating primary key list\n"); + if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) + generate_primary_key_list(mysql, eptr); + + run_scheduler(sptr, query_statements, current, client_limit); + + /* We are finished with this run */ + if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) + drop_primary_key_list(); + } + + if (verbose >= 2) + printf("Generating stats\n"); + + generate_stats(&conclusion, eptr, head_sptr); + + if (!opt_silent) + print_conclusions(&conclusion); + if (opt_csv_str) + print_conclusions_csv(&conclusion); + + my_free((gptr)head_sptr, MYF(0)); + +} + static struct my_option my_long_options[] = { @@ -534,9 +547,6 @@ static struct my_option my_long_options[] = REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"iterations", 'i', "Number of times too run the tests.", (gptr*) &iterations, (gptr*) &iterations, 0, GET_UINT, REQUIRED_ARG, 1, 0, 0, 0, 0, 0}, - {"lock-directory", OPT_MYSQL_LOCK_DIRECTORY, "Directory to use to keep locks.", - (gptr*) &lock_directory, (gptr*) &lock_directory, 0, GET_STR, - REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"number-char-cols", 'x', "Number of VARCHAR columns to create table with if specifying --auto-generate-sql ", (gptr*) &num_char_cols_opt, (gptr*) &num_char_cols_opt, 0, GET_STR, REQUIRED_ARG, @@ -584,17 +594,10 @@ static struct my_option my_long_options[] = {"silent", 's', "Run program in silent mode - no output.", (gptr*) &opt_silent, (gptr*) &opt_silent, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"slave", OPT_MYSQL_SLAP_SLAVE, "Follow master locks for other slap clients", - (gptr*) &opt_slave, (gptr*) &opt_slave, 0, GET_BOOL, NO_ARG, - 0, 0, 0, 0, 0, 0}, {"socket", 'S', "Socket file to use for connection.", (gptr*) &opt_mysql_unix_port, (gptr*) &opt_mysql_unix_port, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, #include - {"use-threads", OPT_USE_THREADS, - "Use pthread calls instead of fork() calls (default on Windows)", - (gptr*) &opt_use_threads, (gptr*) &opt_use_threads, 0, - GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, #ifndef DONT_ALLOW_USER_CHANGE {"user", 'u', "User for login if not current user.", (gptr*) &user, (gptr*) &user, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -1126,11 +1129,6 @@ get_options(int *argc,char ***argv) parse_comma(concurrency_str ? concurrency_str : "1", &concurrency); - if (lock_directory) - snprintf(lock_file_str, FN_REFLEN, "%s/%s", lock_directory, MYSLAPLOCK); - else - snprintf(lock_file_str, FN_REFLEN, "%s/%s", MYSLAPLOCK_DIR, MYSLAPLOCK); - if (opt_csv_str) { opt_silent= TRUE; @@ -1553,136 +1551,63 @@ drop_schema(MYSQL *mysql, const char *db) static int run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) { -#ifndef __WIN__ uint x; -#endif - File lock_file; struct timeval start_time, end_time; thread_context con; DBUG_ENTER("run_scheduler"); con.stmt= stmts; con.limit= limit; - con.thread= opt_use_threads ? 1 :0; - lock_file= my_open(lock_file_str, O_CREAT|O_WRONLY|O_TRUNC, MYF(0)); + pthread_t mainthread; /* Thread descriptor */ + pthread_attr_t attr; /* Thread attributes */ - if (!opt_slave) - if (my_lock(lock_file, F_WRLCK, 0, F_TO_EOF, MYF(0))) + pthread_mutex_lock(&counter_mutex); + thread_counter= 0; + + pthread_mutex_lock(&sleeper_mutex); + master_wakeup= 1; + pthread_mutex_unlock(&sleeper_mutex); + for (x= 0; x < concur; x++) + { + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, + PTHREAD_CREATE_DETACHED); + + /* now create the thread */ + if (pthread_create(&mainthread, &attr, (void *)run_task, + (void *)&con) != 0) { - fprintf(stderr,"%s: Could not get lockfile\n", + fprintf(stderr,"%s: Could not create thread\n", my_progname); exit(0); } - -#ifdef HAVE_LIBPTHREAD - if (opt_use_threads) - { - pthread_t mainthread; /* Thread descriptor */ - pthread_attr_t attr; /* Thread attributes */ - - for (x= 0; x < concur; x++) - { - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, - PTHREAD_CREATE_DETACHED); - - /* now create the thread */ - if (pthread_create(&mainthread, &attr, (void *)run_task, - (void *)&con) != 0) - { - fprintf(stderr,"%s: Could not create thread\n", - my_progname); - exit(0); - } - } + thread_counter++; } -#endif -#if !(defined(__WIN__) || defined(__NETWARE__)) -#ifdef HAVE_LIBPTHREAD - else -#endif - { - fflush(NULL); - for (x= 0; x < concur; x++) - { - int pid; - DBUG_PRINT("info", ("x: %d concurrency: %u", x, *concurrency)); - pid= fork(); - switch(pid) - { - case 0: - /* child */ - DBUG_PRINT("info", ("fork returned 0, calling task(\"%s\"), pid %d gid %d", - stmts ? stmts->string : "", pid, getgid())); - if (verbose >= 3) - printf("%s: fork returned 0, calling task pid %d gid %d\n", - my_progname, pid, getgid()); - run_task(&con); - exit(0); - break; - case -1: - /* error */ - DBUG_PRINT("info", - ("fork returned -1, failing pid %d gid %d", pid, getgid())); - fprintf(stderr, - "%s: Failed on fork: -1, max procs per parent exceeded.\n", - my_progname); - /*exit(1);*/ - goto WAIT; - default: - /* parent, forked */ - DBUG_PRINT("info", ("default, break: pid %d gid %d", pid, getgid())); - if (verbose >= 3) - printf("%s: fork returned %d, gid %d\n", - my_progname, pid, getgid()); - break; - } - } - } -#endif + pthread_mutex_unlock(&counter_mutex); - /* Lets release use some clients! */ - if (!opt_slave) - my_lock(lock_file, F_UNLCK, 0, F_TO_EOF, MYF(0)); + pthread_mutex_lock(&sleeper_mutex); + master_wakeup= 0; + pthread_mutex_unlock(&sleeper_mutex); + pthread_cond_broadcast(&sleep_threshhold); gettimeofday(&start_time, NULL); /* - We look to grab a write lock at this point. Once we get it we know that - all clients have completed their work. + We loop until we know that all children have cleaned up. */ - if (opt_use_threads) + pthread_mutex_lock(&counter_mutex); + while (thread_counter) { - if (my_lock(lock_file, F_WRLCK, 0, F_TO_EOF, MYF(0))) - { - fprintf(stderr,"%s: Could not get lockfile\n", - my_progname); - exit(0); - } - my_lock(lock_file, F_UNLCK, 0, F_TO_EOF, MYF(0)); + struct timespec abstime; + + set_timespec(abstime, 3); + pthread_cond_timedwait(&count_threshhold, &counter_mutex, &abstime); } -#ifndef __WIN__ - else - { -WAIT: - while (x--) - { - int status, pid; - pid= wait(&status); - DBUG_PRINT("info", ("Parent: child %d status %d", pid, status)); - if (status != 0) - { - printf("%s: Child %d died with the status %d\n", - my_progname, pid, status); - exit(0); - } - } - } -#endif + pthread_mutex_unlock(&counter_mutex); + gettimeofday(&end_time, NULL); - my_close(lock_file, MYF(0)); sptr->timing= timedif(end_time, start_time); sptr->users= concur; @@ -1696,7 +1621,6 @@ int run_task(thread_context *con) { ulonglong counter= 0, queries; - File lock_file= -1; MYSQL *mysql; MYSQL_RES *result; MYSQL_ROW row; @@ -1705,6 +1629,13 @@ run_task(thread_context *con) DBUG_ENTER("run_task"); DBUG_PRINT("info", ("task script \"%s\"", con->stmt ? con->stmt->string : "")); + pthread_mutex_lock(&sleeper_mutex); + while (master_wakeup) + { + pthread_cond_wait(&sleep_threshhold, &sleeper_mutex); + } + pthread_mutex_unlock(&sleeper_mutex); + if (!(mysql= mysql_init(NULL))) { fprintf(stderr,"%s: mysql_init() failed ERROR : %s\n", @@ -1712,7 +1643,7 @@ run_task(thread_context *con) exit(0); } - if (con->thread && mysql_thread_init()) + if (mysql_thread_init()) { fprintf(stderr,"%s: mysql_thread_init() failed ERROR : %s\n", my_progname, mysql_error(mysql)); @@ -1720,8 +1651,7 @@ run_task(thread_context *con) } DBUG_PRINT("info", ("trying to connect to host %s as user %s", host, user)); - lock_file= my_open(lock_file_str, O_RDWR, MYF(0)); - my_lock(lock_file, F_RDLCK, 0, F_TO_EOF, MYF(0)); + if (!opt_only_print) { /* Connect to server */ @@ -1821,17 +1751,16 @@ limit_not_met: end: - if (lock_file != -1) - { - my_lock(lock_file, F_UNLCK, 0, F_TO_EOF, MYF(0)); - my_close(lock_file, MYF(0)); - } - if (!opt_only_print) mysql_close(mysql); - if (con->thread) - my_thread_end(); + my_thread_end(); + + pthread_mutex_lock(&counter_mutex); + thread_counter--; + pthread_cond_signal(&count_threshhold); + pthread_mutex_unlock(&counter_mutex); + DBUG_RETURN(0); } diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 3e51bee7c8a..ab40f050aa7 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -1877,8 +1877,7 @@ sub environment_setup () { mtr_native_path($exe_mysqlslap) . " -uroot " . "--port=$master->[0]->{'port'} " . - "--socket=$master->[0]->{'path_sock'} --password= " . - "--lock-directory=$opt_tmpdir"; + "--socket=$master->[0]->{'path_sock'} --password= "; if ( $opt_debug ) { diff --git a/mysql-test/t/mysqlslap.test b/mysql-test/t/mysqlslap.test index f2bb9a72ecf..a40a01f1f25 100644 --- a/mysql-test/t/mysqlslap.test +++ b/mysql-test/t/mysqlslap.test @@ -4,7 +4,7 @@ --exec $MYSQL_SLAP --silent --concurrency=5 --iterations=20 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql ---exec $MYSQL_SLAP --silent --concurrency=5 --iterations=20 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --use-threads +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=20 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --exec $MYSQL_SLAP --only-print --iterations=20 --query="select * from t1" --create="CREATE TABLE t1 (id int, name varchar(64)); INSERT INTO t1 VALUES (1, 'This is a test')" --delimiter=";" --exec $MYSQL_SLAP --silent --concurrency=5 --iterations=20 --query="select * from t1" --create="CREATE TABLE t1 (id int, name varchar(64)); INSERT INTO t1 VALUES (1, 'This is a test')" --delimiter=";" From 786f9de976a8db415db4e9b21f680b5e5a191fca Mon Sep 17 00:00:00 2001 From: "brian@zim.(none)" <> Date: Fri, 16 Mar 2007 19:05:11 -0700 Subject: [PATCH 07/28] Cleanup of prototype for windows. --- client/mysqlslap.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/client/mysqlslap.c b/client/mysqlslap.c index 1b4692848b9..62ee2b03359 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -248,7 +248,7 @@ static int create_schema(MYSQL *mysql, const char *db, statement *stmt, option_string *engine_stmt); static int run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit); -int run_task(thread_context *con); +pthread_handler_t run_task(void *p); void statement_cleanup(statement *stmt); void option_cleanup(option_string *stmt); void concurrency_loop(MYSQL *mysql, uint current, option_string *eptr); @@ -1562,6 +1562,11 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) pthread_t mainthread; /* Thread descriptor */ pthread_attr_t attr; /* Thread attributes */ + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, + PTHREAD_CREATE_DETACHED); + + pthread_mutex_lock(&counter_mutex); thread_counter= 0; @@ -1570,12 +1575,8 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) pthread_mutex_unlock(&sleeper_mutex); for (x= 0; x < concur; x++) { - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, - PTHREAD_CREATE_DETACHED); - - /* now create the thread */ - if (pthread_create(&mainthread, &attr, (void *)run_task, + /* nowucreate the thread */ + if (pthread_create(&mainthread, &attr, run_task, (void *)&con) != 0) { fprintf(stderr,"%s: Could not create thread\n", @@ -1605,6 +1606,7 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) pthread_cond_timedwait(&count_threshhold, &counter_mutex, &abstime); } pthread_mutex_unlock(&counter_mutex); + pthread_attr_destroy(&attr); gettimeofday(&end_time, NULL); @@ -1617,14 +1619,14 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) } -int -run_task(thread_context *con) +pthread_handler_t run_task(void *p) { ulonglong counter= 0, queries; MYSQL *mysql; MYSQL_RES *result; MYSQL_ROW row; statement *ptr; + thread_context *con= (thread_context *)p; DBUG_ENTER("run_task"); DBUG_PRINT("info", ("task script \"%s\"", con->stmt ? con->stmt->string : "")); From 34b2658788c85e71961b40169933c2823967b438 Mon Sep 17 00:00:00 2001 From: "brian@zim.(none)" <> Date: Fri, 16 Mar 2007 23:50:48 -0700 Subject: [PATCH 08/28] Bug in windows where attr was not working. Removed. --- client/mysqlslap.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/client/mysqlslap.c b/client/mysqlslap.c index 62ee2b03359..de7a726747e 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -1560,12 +1560,6 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) con.limit= limit; pthread_t mainthread; /* Thread descriptor */ - pthread_attr_t attr; /* Thread attributes */ - - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, - PTHREAD_CREATE_DETACHED); - pthread_mutex_lock(&counter_mutex); thread_counter= 0; @@ -1576,7 +1570,7 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) for (x= 0; x < concur; x++) { /* nowucreate the thread */ - if (pthread_create(&mainthread, &attr, run_task, + if (pthread_create(&mainthread, NULL, run_task, (void *)&con) != 0) { fprintf(stderr,"%s: Could not create thread\n", @@ -1606,7 +1600,6 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) pthread_cond_timedwait(&count_threshhold, &counter_mutex, &abstime); } pthread_mutex_unlock(&counter_mutex); - pthread_attr_destroy(&attr); gettimeofday(&end_time, NULL); @@ -1793,13 +1786,13 @@ parse_option(const char *origin, option_string **stmt, char delm) char *option_ptr; tmp->length= (size_t)(buffer_ptr - buffer); - tmp->string= my_strndup(ptr, tmp->length, MYF(MY_FAE)); + tmp->string= my_strndup(ptr, (uint)tmp->length, MYF(MY_FAE)); option_ptr= ptr + 1 + tmp->length; /* Move past the : and the first string */ tmp->option_length= (size_t)(retstr - option_ptr); - tmp->option= my_strndup(option_ptr, tmp->option_length, + tmp->option= my_strndup(option_ptr, (uint)tmp->option_length, MYF(MY_FAE)); } else @@ -1863,7 +1856,7 @@ parse_delimiter(const char *script, statement **stmt, char delm) tmp= tmp->next) { count++; - tmp->string= my_strndup(ptr, (size_t)(retstr - ptr), MYF(MY_FAE)); + tmp->string= my_strndup(ptr, (uint)(retstr - ptr), MYF(MY_FAE)); tmp->length= (size_t)(retstr - ptr); ptr+= retstr - ptr + 1; if (isspace(*ptr)) @@ -1873,7 +1866,7 @@ parse_delimiter(const char *script, statement **stmt, char delm) if (ptr != script+length) { - tmp->string= my_strndup(ptr, (size_t)((script + length) - ptr), + tmp->string= my_strndup(ptr, (uint)((script + length) - ptr), MYF(MY_FAE)); tmp->length= (size_t)((script + length) - ptr); count++; @@ -1942,7 +1935,7 @@ print_conclusions_csv(conclusions *con) con->users, /* Children used */ con->avg_rows /* Queries run */ ); - my_write(csv_file, buffer, strlen(buffer), MYF(0)); + my_write(csv_file, buffer, (uint)strlen(buffer), MYF(0)); } void From 132608419ef5b495d59e6418f2305ceb24a1e9c6 Mon Sep 17 00:00:00 2001 From: "brian@piggy.tangent.org" <> Date: Sun, 18 Mar 2007 10:06:28 -0700 Subject: [PATCH 09/28] Restoring attr (aka poking around Windows did not work). --- client/mysqlslap.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/client/mysqlslap.c b/client/mysqlslap.c index de7a726747e..6fb810b6a23 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -1560,6 +1560,10 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) con.limit= limit; pthread_t mainthread; /* Thread descriptor */ + pthread_attr_t attr; /* Thread attributes */ + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, + PTHREAD_CREATE_DETACHED); pthread_mutex_lock(&counter_mutex); thread_counter= 0; @@ -1570,7 +1574,7 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) for (x= 0; x < concur; x++) { /* nowucreate the thread */ - if (pthread_create(&mainthread, NULL, run_task, + if (pthread_create(&mainthread, &attr, run_task, (void *)&con) != 0) { fprintf(stderr,"%s: Could not create thread\n", @@ -1580,6 +1584,7 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) thread_counter++; } pthread_mutex_unlock(&counter_mutex); + pthread_attr_destroy(&attr); pthread_mutex_lock(&sleeper_mutex); master_wakeup= 0; From 9da24df4e03f417ee2fe857ae93081dbbf327bc5 Mon Sep 17 00:00:00 2001 From: "brian@zim.(none)" <> Date: Tue, 20 Mar 2007 22:13:55 -0700 Subject: [PATCH 10/28] Modified read buffer to see if performance difference existed. Re-enabled 8gig unit test. --- storage/archive/archive_test.c | 16 +++++++++------- storage/archive/azio.c | 31 ++++++++++++++++--------------- storage/archive/azlib.h | 7 ++++--- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/storage/archive/archive_test.c b/storage/archive/archive_test.c index 9ac043330fc..a5b2d1dfcc9 100644 --- a/storage/archive/archive_test.c +++ b/storage/archive/archive_test.c @@ -217,14 +217,13 @@ int main(int argc, char *argv[]) azclose(&writer_handle); azclose(&reader_handle); - exit(0); unlink(TEST_FILENAME); /* Start size tests */ printf("About to run 2/4/8 gig tests now, you may want to hit CTRL-C\n"); - size_test(TWOGIG, 2097152L); - size_test(FOURGIG, 4194304L); - size_test(EIGHTGIG, 8388608L); + size_test(TWOGIG, 2088992L); + size_test(FOURGIG, 4177984L); + size_test(EIGHTGIG, 8355968L); return 0; } @@ -234,6 +233,7 @@ int size_test(unsigned long long length, unsigned long long rows_to_test_for) azio_stream writer_handle, reader_handle; unsigned long long write_length; unsigned long long read_length= 0; + unsigned long long count; unsigned int ret; char buffer[BUFFER_LEN]; int error; @@ -244,8 +244,10 @@ int size_test(unsigned long long length, unsigned long long rows_to_test_for) return 0; } - for (write_length= 0; write_length < length ; write_length+= ret) + for (count= 0, write_length= 0; write_length < length ; + write_length+= ret) { + count++; ret= azwrite(&writer_handle, test_string, BUFFER_LEN); if (ret != BUFFER_LEN) { @@ -257,7 +259,7 @@ int size_test(unsigned long long length, unsigned long long rows_to_test_for) azflush(&writer_handle, Z_SYNC_FLUSH); } } - assert(write_length == length); + assert(write_length != count * BUFFER_LEN); /* Number of rows time BUFFER_LEN */ azflush(&writer_handle, Z_SYNC_FLUSH); printf("Reading back data\n"); @@ -279,7 +281,7 @@ int size_test(unsigned long long length, unsigned long long rows_to_test_for) } } - assert(read_length == length); + assert(read_length == write_length); assert(writer_handle.rows == rows_to_test_for); azclose(&writer_handle); azclose(&reader_handle); diff --git a/storage/archive/azio.c b/storage/archive/azio.c index 7876dd69cab..6b01d9c3c88 100644 --- a/storage/archive/azio.c +++ b/storage/archive/azio.c @@ -55,8 +55,8 @@ int az_open (azio_stream *s, const char *path, int Flags, File fd) s->stream.zalloc = (alloc_func)0; s->stream.zfree = (free_func)0; s->stream.opaque = (voidpf)0; - memset(s->inbuf, 0, AZ_BUFSIZE); - memset(s->outbuf, 0, AZ_BUFSIZE); + memset(s->inbuf, 0, AZ_BUFSIZE_READ); + memset(s->outbuf, 0, AZ_BUFSIZE_WRITE); s->stream.next_in = s->inbuf; s->stream.next_out = s->outbuf; s->stream.avail_in = s->stream.avail_out = 0; @@ -109,7 +109,7 @@ int az_open (azio_stream *s, const char *path, int Flags, File fd) return Z_NULL; } } - s->stream.avail_out = AZ_BUFSIZE; + s->stream.avail_out = AZ_BUFSIZE_WRITE; errno = 0; s->file = fd < 0 ? my_open(path, Flags, MYF(0)) : fd; @@ -159,7 +159,7 @@ void write_header(azio_stream *s) char buffer[AZHEADER_SIZE + AZMETA_BUFFER_SIZE]; char *ptr= buffer; - s->block_size= AZ_BUFSIZE; + s->block_size= AZ_BUFSIZE_WRITE; s->version = (unsigned char)az_magic[1]; s->minor_version = (unsigned char)az_magic[2]; @@ -224,7 +224,7 @@ int get_byte(s) if (s->stream.avail_in == 0) { errno = 0; - s->stream.avail_in = my_read(s->file, (byte *)s->inbuf, AZ_BUFSIZE, MYF(0)); + s->stream.avail_in = my_read(s->file, (byte *)s->inbuf, AZ_BUFSIZE_READ, MYF(0)); if (s->stream.avail_in == 0) { s->z_eof = 1; @@ -260,7 +260,7 @@ void check_header(azio_stream *s) if (len < 2) { if (len) s->inbuf[0] = s->stream.next_in[0]; errno = 0; - len = (uInt)my_read(s->file, (byte *)s->inbuf + len, AZ_BUFSIZE >> len, MYF(0)); + len = (uInt)my_read(s->file, (byte *)s->inbuf + len, AZ_BUFSIZE_READ >> len, MYF(0)); if (len == 0) s->z_err = Z_ERRNO; s->stream.avail_in += len; s->stream.next_in = s->inbuf; @@ -455,7 +455,7 @@ unsigned int ZEXPORT azread ( azio_stream *s, voidp buf, unsigned int len, int * if (s->stream.avail_in == 0 && !s->z_eof) { errno = 0; - s->stream.avail_in = (uInt)my_read(s->file, (byte *)s->inbuf, AZ_BUFSIZE, MYF(0)); + s->stream.avail_in = (uInt)my_read(s->file, (byte *)s->inbuf, AZ_BUFSIZE_READ, MYF(0)); if (s->stream.avail_in == 0) { s->z_eof = 1; @@ -522,12 +522,13 @@ unsigned int azwrite (azio_stream *s, voidpc buf, unsigned int len) { s->stream.next_out = s->outbuf; - if (my_write(s->file, (byte *)s->outbuf, AZ_BUFSIZE, MYF(0)) != AZ_BUFSIZE) + if (my_write(s->file, (byte *)s->outbuf, AZ_BUFSIZE_WRITE, + MYF(0)) != AZ_BUFSIZE_WRITE) { s->z_err = Z_ERRNO; break; } - s->stream.avail_out = AZ_BUFSIZE; + s->stream.avail_out = AZ_BUFSIZE_WRITE; } s->in += s->stream.avail_in; s->out += s->stream.avail_out; @@ -563,7 +564,7 @@ int do_flush (azio_stream *s, int flush) for (;;) { - len = AZ_BUFSIZE - s->stream.avail_out; + len = AZ_BUFSIZE_WRITE - s->stream.avail_out; if (len != 0) { @@ -574,7 +575,7 @@ int do_flush (azio_stream *s, int flush) return Z_ERRNO; } s->stream.next_out = s->outbuf; - s->stream.avail_out = AZ_BUFSIZE; + s->stream.avail_out = AZ_BUFSIZE_WRITE; } if (done) break; s->out += s->stream.avail_out; @@ -675,8 +676,8 @@ my_off_t azseek (s, offset, whence) /* There was a zmemzero here if inbuf was null -Brian */ while (offset > 0) { - uInt size = AZ_BUFSIZE; - if (offset < AZ_BUFSIZE) size = (uInt)offset; + uInt size = AZ_BUFSIZE_WRITE; + if (offset < AZ_BUFSIZE_WRITE) size = (uInt)offset; size = azwrite(s, s->inbuf, size); if (size == 0) return -1L; @@ -719,8 +720,8 @@ my_off_t azseek (s, offset, whence) } while (offset > 0) { int error; - unsigned int size = AZ_BUFSIZE; - if (offset < AZ_BUFSIZE) size = (int)offset; + unsigned int size = AZ_BUFSIZE_READ; + if (offset < AZ_BUFSIZE_READ) size = (int)offset; size = azread(s, s->outbuf, size, &error); if (error <= 0) return -1L; diff --git a/storage/archive/azlib.h b/storage/archive/azlib.h index 9076ff23192..663b746ec52 100644 --- a/storage/archive/azlib.h +++ b/storage/archive/azlib.h @@ -196,7 +196,8 @@ extern "C" { /* The deflate compression method (the only one supported in this version) */ #define Z_NULL 0 /* for initializing zalloc, zfree, opaque */ -#define AZ_BUFSIZE 16384 +#define AZ_BUFSIZE_READ 524288 +#define AZ_BUFSIZE_WRITE 16384 typedef struct azio_stream { @@ -204,8 +205,8 @@ typedef struct azio_stream { int z_err; /* error code for last stream operation */ int z_eof; /* set if end of input file */ File file; /* .gz file */ - Byte inbuf[AZ_BUFSIZE]; /* input buffer */ - Byte outbuf[AZ_BUFSIZE]; /* output buffer */ + Byte inbuf[AZ_BUFSIZE_READ]; /* input buffer */ + Byte outbuf[AZ_BUFSIZE_WRITE]; /* output buffer */ uLong crc; /* crc32 of uncompressed data */ char *msg; /* error message */ int transparent; /* 1 if input file is not a .gz file */ From adfdcb191ca1a0d48db3c760ea186e0502664310 Mon Sep 17 00:00:00 2001 From: "Antony@floppy." <> Date: Thu, 22 Mar 2007 00:43:14 -0700 Subject: [PATCH 11/28] Fixes for 5.1-arch repository to compile on Windows. --- CMakeLists.txt | 2 +- client/CMakeLists.txt | 1 + client/mysqlslap.c | 8 ++------ 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eb4d77db73f..48992cf574b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -131,10 +131,10 @@ ADD_SUBDIRECTORY(dbug) ADD_SUBDIRECTORY(strings) ADD_SUBDIRECTORY(regex) ADD_SUBDIRECTORY(mysys) +ADD_SUBDIRECTORY(zlib) ADD_SUBDIRECTORY(extra/yassl) ADD_SUBDIRECTORY(extra/yassl/taocrypt) ADD_SUBDIRECTORY(extra) -ADD_SUBDIRECTORY(zlib) ADD_SUBDIRECTORY(storage/heap) ADD_SUBDIRECTORY(storage/myisam) ADD_SUBDIRECTORY(storage/myisammrg) diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index d492c49a6b2..ea9e7547354 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -95,6 +95,7 @@ ADD_EXECUTABLE(mysqladmin mysqladmin.cc) TARGET_LINK_LIBRARIES(mysqladmin mysqlclient mysys dbug yassl taocrypt zlib wsock32) ADD_EXECUTABLE(mysqlslap mysqlslap.c) +SET_SOURCE_FILES_PROPERTIES(mysqlslap.c PROPERTIES COMPILE_FLAGS "-DTHREADS") TARGET_LINK_LIBRARIES(mysqlslap mysqlclient mysys yassl taocrypt zlib wsock32 dbug) ADD_EXECUTABLE(echo echo.c) diff --git a/client/mysqlslap.c b/client/mysqlslap.c index 6fb810b6a23..809c1a6b5da 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -82,10 +82,6 @@ TODO: #define SELECT_TYPE_REQUIRES_PREFIX 5 #include "client_priv.h" -#include -#include -#include -#include #include #include #include @@ -1554,13 +1550,13 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) uint x; struct timeval start_time, end_time; thread_context con; + pthread_t mainthread; /* Thread descriptor */ + pthread_attr_t attr; /* Thread attributes */ DBUG_ENTER("run_scheduler"); con.stmt= stmts; con.limit= limit; - pthread_t mainthread; /* Thread descriptor */ - pthread_attr_t attr; /* Thread attributes */ pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); From 2527b6ba1b34fad88e646226fd4deb1aba52beac Mon Sep 17 00:00:00 2001 From: "acurtis/antony@xiphis.org/ltamd64.xiphis.org" <> Date: Thu, 22 Mar 2007 01:59:47 -0700 Subject: [PATCH 12/28] fixes for 5.1-arch repository --- client/Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/client/Makefile.am b/client/Makefile.am index 1c908ea715e..4b78ee2b0c9 100644 --- a/client/Makefile.am +++ b/client/Makefile.am @@ -76,6 +76,7 @@ mysqlimport_LDADD = $(CXXLDFLAGS) $(CLIENT_THREAD_LIBS) \ mysqlshow_SOURCES= mysqlshow.c mysqlslap_SOURCES= mysqlslap.c +mysqlslap_CFLAGS= -DTHREAD -UUNDEF_THREADS_HACK mysqlslap_LDADD = $(CXXLDFLAGS) $(CLIENT_THREAD_LIBS) \ @CLIENT_EXTRA_LDFLAGS@ \ $(LIBMYSQLCLIENT_LA) \ From 2220fe2774bf2b041ba9894e33d23d2bf03a5df1 Mon Sep 17 00:00:00 2001 From: "serg@sergbook.mysql.com" <> Date: Fri, 23 Mar 2007 13:38:42 +0200 Subject: [PATCH 13/28] move thr_client_alarm initialization to mysqld.cc (in thr_alarm.cc it happened too late). --- mysys/thr_alarm.c | 4 ---- sql/mysqld.cc | 14 +++++++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/mysys/thr_alarm.c b/mysys/thr_alarm.c index 759544af17b..396623dea21 100644 --- a/mysys/thr_alarm.c +++ b/mysys/thr_alarm.c @@ -78,10 +78,6 @@ void init_thr_alarm(uint max_alarms) sigfillset(&full_signal_set); /* Neaded to block signals */ pthread_mutex_init(&LOCK_alarm,MY_MUTEX_INIT_FAST); pthread_cond_init(&COND_alarm,NULL); - if (thd_lib_detected == THD_LIB_LT) - thr_client_alarm= SIGALRM; - else - thr_client_alarm= SIGUSR1; #ifndef USE_ALARM_THREAD if (thd_lib_detected != THD_LIB_LT) #endif diff --git a/sql/mysqld.cc b/sql/mysqld.cc index a3263b50951..0e9131389c9 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -2046,6 +2046,17 @@ static void init_signals(void) struct sigaction sa; DBUG_ENTER("init_signals"); + if (thd_lib_detected == THD_LIB_LT) + { + thr_client_alarm= SIGALRM; + thr_kill_signal= SIGINT; + } + else + { + thr_client_alarm= SIGUSR1; + thr_kill_signal= SIGUSR2; + } + if (test_flags & TEST_SIGINT) { my_sigset(thr_kill_signal, end_thread_signal); @@ -3140,9 +3151,6 @@ int main(int argc, char **argv) MY_INIT(argv[0]); // init my_sys library & pthreads - /* Set signal used to kill MySQL */ - thr_kill_signal= thd_lib_detected == THD_LIB_LT ? SIGINT : SIGUSR2; - #ifdef _CUSTOMSTARTUPCONFIG_ if (_cust_check_startup()) { From d2d72bfc586d3dc1ca28ad03e2ef1853e08af52e Mon Sep 17 00:00:00 2001 From: "serg@sergbook.mysql.com" <> Date: Fri, 23 Mar 2007 20:37:20 +0200 Subject: [PATCH 14/28] reverted linuxthreads thr_client_alarm fix (not future-proof) fixed differently: wake up select_thread with THR_SERVER_ALARM instead --- mysys/thr_alarm.c | 4 ++++ sql/mysqld.cc | 23 ++++++++--------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/mysys/thr_alarm.c b/mysys/thr_alarm.c index 396623dea21..759544af17b 100644 --- a/mysys/thr_alarm.c +++ b/mysys/thr_alarm.c @@ -78,6 +78,10 @@ void init_thr_alarm(uint max_alarms) sigfillset(&full_signal_set); /* Neaded to block signals */ pthread_mutex_init(&LOCK_alarm,MY_MUTEX_INIT_FAST); pthread_cond_init(&COND_alarm,NULL); + if (thd_lib_detected == THD_LIB_LT) + thr_client_alarm= SIGALRM; + else + thr_client_alarm= SIGUSR1; #ifndef USE_ALARM_THREAD if (thd_lib_detected != THD_LIB_LT) #endif diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 903d49a468c..e26bd31a00a 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -622,7 +622,7 @@ static void close_connections(void) DBUG_PRINT("info",("Waiting for select thread")); #ifndef DONT_USE_THR_ALARM - if (pthread_kill(select_thread, thr_client_alarm)) + if (pthread_kill(select_thread, THR_SERVER_ALARM)) break; // allready dead #endif set_timespec(abstime, 2); @@ -2062,17 +2062,6 @@ static void init_signals(void) struct sigaction sa; DBUG_ENTER("init_signals"); - if (thd_lib_detected == THD_LIB_LT) - { - thr_client_alarm= SIGALRM; - thr_kill_signal= SIGINT; - } - else - { - thr_client_alarm= SIGUSR1; - thr_kill_signal= SIGUSR2; - } - if (test_flags & TEST_SIGINT) { my_sigset(thr_kill_signal, end_thread_signal); @@ -2131,14 +2120,11 @@ static void init_signals(void) #ifdef SIGTSTP sigaddset(&set,SIGTSTP); #endif - sigaddset(&set,THR_SERVER_ALARM); if (test_flags & TEST_SIGINT) { // May be SIGINT sigdelset(&set, thr_kill_signal); } - // For alarms - sigdelset(&set, thr_client_alarm); sigprocmask(SIG_SETMASK,&set,NULL); pthread_sigmask(SIG_SETMASK,&set,NULL); DBUG_VOID_RETURN; @@ -3167,6 +3153,13 @@ int main(int argc, char **argv) MY_INIT(argv[0]); // init my_sys library & pthreads + /* Set signal used to kill MySQL */ +#if defined(SIGUSR2) + thr_kill_signal= thd_lib_detected == THD_LIB_LT ? SIGINT : SIGUSR2; +#else + thr_kill_signal= thd_lib_detected == SIGINT; +#endif + #ifdef _CUSTOMSTARTUPCONFIG_ if (_cust_check_startup()) { From 77718a3fc19e0e7854a94c4aad3342b04053977c Mon Sep 17 00:00:00 2001 From: "serg@sergbook.mysql.com" <> Date: Fri, 23 Mar 2007 22:16:32 +0200 Subject: [PATCH 15/28] bug in im* tests - missing --no-defaults in the mysql command line --- mysql-test/t/wait_for_socket.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql-test/t/wait_for_socket.sh b/mysql-test/t/wait_for_socket.sh index 8c17c8ac0ac..2fa7d5c5b7e 100755 --- a/mysql-test/t/wait_for_socket.sh +++ b/mysql-test/t/wait_for_socket.sh @@ -61,7 +61,7 @@ fi ########################################################################### -client_args="--silent --socket=$socket_path --connect_timeout=1 " +client_args="--no-defaults --silent --socket=$socket_path --connect_timeout=1 " [ -n "$username" ] && client_args="$client_args --user=$username " [ -n "$password" ] && client_args="$client_args --password=$password " From bbe97fa4e3adff4d85650eb3f372183e63d79b26 Mon Sep 17 00:00:00 2001 From: "brian@zim.(none)" <> Date: Fri, 23 Mar 2007 18:51:56 -0700 Subject: [PATCH 16/28] Added delayed open of file descriptors to cut down on issues surrounding large collections of partitions being open by many threads. --- storage/archive/azlib.h | 2 +- storage/archive/ha_archive.cc | 44 +++++++++++++++++++++++++---------- storage/archive/ha_archive.h | 2 ++ 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/storage/archive/azlib.h b/storage/archive/azlib.h index 663b746ec52..a5bee1befae 100644 --- a/storage/archive/azlib.h +++ b/storage/archive/azlib.h @@ -196,7 +196,7 @@ extern "C" { /* The deflate compression method (the only one supported in this version) */ #define Z_NULL 0 /* for initializing zalloc, zfree, opaque */ -#define AZ_BUFSIZE_READ 524288 +#define AZ_BUFSIZE_READ 32768 #define AZ_BUFSIZE_WRITE 16384 diff --git a/storage/archive/ha_archive.cc b/storage/archive/ha_archive.cc index bb638e1c17b..307f6665c10 100644 --- a/storage/archive/ha_archive.cc +++ b/storage/archive/ha_archive.cc @@ -210,7 +210,8 @@ ha_archive::ha_archive(handlerton *hton, TABLE_SHARE *table_arg) buffer.set((char *)byte_buffer, IO_SIZE, system_charset_info); /* The size of the offset value we will use for position() */ - ref_length = sizeof(my_off_t); + ref_length= sizeof(my_off_t); + archive_reader_open= FALSE; } int archive_discover(handlerton *hton, THD* thd, const char *db, @@ -434,6 +435,29 @@ int ha_archive::init_archive_writer() } +int ha_archive::init_archive_reader() +{ + DBUG_ENTER("ha_archive::init_archive_reader"); + /* + It is expensive to open and close the data files and since you can't have + a gzip file that can be both read and written we keep a writer open + that is shared amoung all open tables. + */ + if (!archive_reader_open) + { + if (!(azopen(&archive, share->data_file_name, O_RDONLY|O_BINARY))) + { + DBUG_PRINT("ha_archive", ("Could not open archive read file")); + share->crashed= TRUE; + DBUG_RETURN(1); + } + archive_reader_open= TRUE; + } + + DBUG_RETURN(0); +} + + /* We just implement one additional file extension. */ @@ -477,7 +501,6 @@ int ha_archive::open(const char *name, int mode, uint open_options) DBUG_ASSERT(share); - record_buffer= create_record_buffer(table->s->reclength + ARCHIVE_ROW_HEADER_SIZE); @@ -489,14 +512,6 @@ int ha_archive::open(const char *name, int mode, uint open_options) thr_lock_data_init(&share->lock, &lock, NULL); - DBUG_PRINT("ha_archive", ("archive data_file_name %s", share->data_file_name)); - if (!(azopen(&archive, share->data_file_name, O_RDONLY|O_BINARY))) - { - if (errno == EROFS || errno == EACCES) - DBUG_RETURN(my_errno= errno); - DBUG_RETURN(HA_ERR_CRASHED_ON_USAGE); - } - DBUG_PRINT("ha_archive", ("archive table was crashed %s", rc == HA_ERR_CRASHED_ON_USAGE ? "yes" : "no")); if (rc == HA_ERR_CRASHED_ON_USAGE && open_options & HA_OPEN_FOR_REPAIR) @@ -533,8 +548,11 @@ int ha_archive::close(void) destroy_record_buffer(record_buffer); /* First close stream */ - if (azclose(&archive)) - rc= 1; + if (archive_reader_open) + { + if (azclose(&archive)) + rc= 1; + } /* then also close share */ rc|= free_share(); @@ -979,6 +997,8 @@ int ha_archive::rnd_init(bool scan) if (share->crashed) DBUG_RETURN(HA_ERR_CRASHED_ON_USAGE); + init_archive_reader(); + /* We rewind the file so that we can read from the beginning if scan */ if (scan) { diff --git a/storage/archive/ha_archive.h b/storage/archive/ha_archive.h index 8f56e8ce060..8fc54f6715f 100644 --- a/storage/archive/ha_archive.h +++ b/storage/archive/ha_archive.h @@ -71,6 +71,7 @@ class ha_archive: public handler uint current_key_len; uint current_k_offset; archive_record_buffer *record_buffer; + bool archive_reader_open; archive_record_buffer *create_record_buffer(unsigned int length); void destroy_record_buffer(archive_record_buffer *r); @@ -119,6 +120,7 @@ public: ARCHIVE_SHARE *get_share(const char *table_name, int *rc); int free_share(); int init_archive_writer(); + int init_archive_reader(); bool auto_repair() const { return 1; } // For the moment we just do this int read_data_header(azio_stream *file_to_read); void position(const byte *record); From 18a01e72619008ad35b749711d5913e03c330968 Mon Sep 17 00:00:00 2001 From: "serg@sergbook.mysql.com" <> Date: Sat, 24 Mar 2007 14:03:27 +0200 Subject: [PATCH 17/28] nptl: typo fixed. sigaddset restored --- sql/mysqld.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index e26bd31a00a..51332053df6 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -622,7 +622,7 @@ static void close_connections(void) DBUG_PRINT("info",("Waiting for select thread")); #ifndef DONT_USE_THR_ALARM - if (pthread_kill(select_thread, THR_SERVER_ALARM)) + if (pthread_kill(select_thread, thr_client_alarm)) break; // allready dead #endif set_timespec(abstime, 2); @@ -2120,6 +2120,8 @@ static void init_signals(void) #ifdef SIGTSTP sigaddset(&set,SIGTSTP); #endif + if (thd_lib_detected != THD_LIB_LT) + sigaddset(&set,THR_SERVER_ALARM); if (test_flags & TEST_SIGINT) { // May be SIGINT @@ -3157,7 +3159,7 @@ int main(int argc, char **argv) #if defined(SIGUSR2) thr_kill_signal= thd_lib_detected == THD_LIB_LT ? SIGINT : SIGUSR2; #else - thr_kill_signal= thd_lib_detected == SIGINT; + thr_kill_signal= SIGINT; #endif #ifdef _CUSTOMSTARTUPCONFIG_ From ff9da1f048470faf8775b4046a5378a9b4e2f64b Mon Sep 17 00:00:00 2001 From: "brian@zim.(none)" <> Date: Sun, 25 Mar 2007 11:38:49 -0700 Subject: [PATCH 18/28] New comments, additional init for rebuild options. --- storage/archive/ha_archive.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/storage/archive/ha_archive.cc b/storage/archive/ha_archive.cc index 307f6665c10..72647b58f54 100644 --- a/storage/archive/ha_archive.cc +++ b/storage/archive/ha_archive.cc @@ -81,6 +81,7 @@ TODO: Allow users to set compression level. + Allow adjustable block size. Implement versioning, should be easy. Allow for errors, find a way to mark bad rows. Add optional feature so that rows can be flushed at interval (which will cause less @@ -1303,6 +1304,8 @@ int ha_archive::optimize(THD* thd, HA_CHECK_OPT* check_opt) azio_stream writer; char writer_filename[FN_REFLEN]; + init_archive_reader(); + // now we close both our writer and our reader for the rename if (share->archive_write_open) { @@ -1577,6 +1580,8 @@ int ha_archive::check(THD* thd, HA_CHECK_OPT* check_opt) Now we will rewind the archive file so that we are positioned at the start of the file. */ + init_archive_reader(); + if (!rc) read_data_header(&archive); From 9a2a7071be3288afc0297f89d88878bd857aa9d5 Mon Sep 17 00:00:00 2001 From: "brian@zim.(none)" <> Date: Mon, 26 Mar 2007 02:24:49 -0700 Subject: [PATCH 19/28] Added pre and post option to test run (allows me to adjust an engine once all of the data has been loaded). Kinda obvious that this was going to come up ;) --- client/client_priv.h | 2 + client/mysqlslap.c | 106 +++++++++++++++++++++++++++++++++- mysql-test/r/mysqlslap.result | 24 ++++++++ mysql-test/t/mysqlslap.test | 2 + 4 files changed, 132 insertions(+), 2 deletions(-) diff --git a/client/client_priv.h b/client/client_priv.h index 6227dcdee44..4dbf3f7895b 100644 --- a/client/client_priv.h +++ b/client/client_priv.h @@ -63,6 +63,8 @@ enum options_client OPT_SLAP_AUTO_GENERATE_SECONDARY_INDEXES, OPT_SLAP_AUTO_GENERATE_UNIQUE_WRITE_NUM, OPT_SLAP_AUTO_GENERATE_UNIQUE_QUERY_NUM, + OPT_SLAP_PRE_QUERY, + OPT_SLAP_POST_QUERY, OPT_MYSQL_REPLACE_INTO, OPT_BASE64_OUTPUT, OPT_SERVER_ID, OPT_FIX_TABLE_NAMES, OPT_FIX_DB_NAMES, OPT_SSL_VERIFY_SERVER_CERT, OPT_DEBUG_INFO, OPT_COLUMN_TYPES diff --git a/client/mysqlslap.c b/client/mysqlslap.c index de7a726747e..e4a70cd8d11 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -122,6 +122,8 @@ unsigned long long primary_keys_number_of; static char *host= NULL, *opt_password= NULL, *user= NULL, *user_supplied_query= NULL, + *user_supplied_pre_statements= NULL, + *user_supplied_post_statements= NULL, *default_engine= NULL, *opt_mysql_unix_port= NULL; @@ -226,6 +228,8 @@ struct conclusions { }; static option_string *engine_options= NULL; +static statement *pre_statements= NULL; +static statement *post_statements= NULL; static statement *create_statements= NULL, *query_statements= NULL; @@ -252,6 +256,7 @@ pthread_handler_t run_task(void *p); void statement_cleanup(statement *stmt); void option_cleanup(option_string *stmt); void concurrency_loop(MYSQL *mysql, uint current, option_string *eptr); +static int run_statements(MYSQL *mysql, statement *stmt); static const char ALPHANUMERICS[]= "0123456789ABCDEFGHIJKLMNOPQRSTWXYZabcdefghijklmnopqrstuvwxyz"; @@ -392,6 +397,8 @@ int main(int argc, char **argv) statement_cleanup(create_statements); statement_cleanup(query_statements); + statement_cleanup(pre_statements); + statement_cleanup(post_statements); option_cleanup(engine_options); #ifdef HAVE_SMEM @@ -447,7 +454,13 @@ void concurrency_loop(MYSQL *mysql, uint current, option_string *eptr) if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) generate_primary_key_list(mysql, eptr); + if (pre_statements) + run_statements(mysql, pre_statements); + run_scheduler(sptr, query_statements, current, client_limit); + + if (post_statements) + run_statements(mysql, post_statements); /* We are finished with this run */ if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) @@ -574,6 +587,16 @@ static struct my_option my_long_options[] = {"port", 'P', "Port number to use for connection.", (gptr*) &opt_mysql_port, (gptr*) &opt_mysql_port, 0, GET_UINT, REQUIRED_ARG, MYSQL_PORT, 0, 0, 0, 0, 0}, + {"post-query", OPT_SLAP_POST_QUERY, + "Query to run or file containing query to run after executing.", + (gptr*) &user_supplied_post_statements, + (gptr*) &user_supplied_post_statements, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"pre-query", OPT_SLAP_PRE_QUERY, + "Query to run or file containing query to run before executing.", + (gptr*) &user_supplied_pre_statements, + (gptr*) &user_supplied_pre_statements, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"preserve-schema", OPT_MYSQL_PRESERVE_SCHEMA, "Preserve the schema from the mysqlslap run, this happens unless " "--auto-generate-sql or --create are used.", @@ -1336,6 +1359,66 @@ get_options(int *argc,char ***argv) } } + if (user_supplied_pre_statements && my_stat(user_supplied_pre_statements, &sbuf, MYF(0))) + { + File data_file; + if (!MY_S_ISREG(sbuf.st_mode)) + { + fprintf(stderr,"%s: User query supplied file was not a regular file\n", + my_progname); + exit(1); + } + if ((data_file= my_open(user_supplied_pre_statements, O_RDWR, MYF(0))) == -1) + { + fprintf(stderr,"%s: Could not open query supplied file\n", my_progname); + exit(1); + } + tmp_string= (char *)my_malloc(sbuf.st_size + 1, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); + my_read(data_file, tmp_string, sbuf.st_size, MYF(0)); + tmp_string[sbuf.st_size]= '\0'; + my_close(data_file,MYF(0)); + if (user_supplied_pre_statements) + actual_queries= parse_delimiter(tmp_string, &pre_statements, + delimiter[0]); + my_free((gptr)tmp_string, MYF(0)); + } + else if (user_supplied_pre_statements) + { + actual_queries= parse_delimiter(user_supplied_pre_statements, &pre_statements, + delimiter[0]); + } + + if (user_supplied_post_statements && my_stat(user_supplied_post_statements, &sbuf, MYF(0))) + { + File data_file; + if (!MY_S_ISREG(sbuf.st_mode)) + { + fprintf(stderr,"%s: User query supplied file was not a regular file\n", + my_progname); + exit(1); + } + if ((data_file= my_open(user_supplied_post_statements, O_RDWR, MYF(0))) == -1) + { + fprintf(stderr,"%s: Could not open query supplied file\n", my_progname); + exit(1); + } + tmp_string= (char *)my_malloc(sbuf.st_size + 1, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); + my_read(data_file, tmp_string, sbuf.st_size, MYF(0)); + tmp_string[sbuf.st_size]= '\0'; + my_close(data_file,MYF(0)); + if (user_supplied_post_statements) + parse_delimiter(tmp_string, &post_statements, + delimiter[0]); + my_free((gptr)tmp_string, MYF(0)); + } + else if (user_supplied_post_statements) + { + parse_delimiter(user_supplied_post_statements, &post_statements, + delimiter[0]); + } + if (verbose >= 2) printf("Parsing engines to use.\n"); @@ -1368,7 +1451,7 @@ generate_primary_key_list(MYSQL *mysql, option_string *engine_stmt) MYSQL_RES *result; MYSQL_ROW row; unsigned long long counter; - DBUG_ENTER("create_schema"); + DBUG_ENTER("generate_primary_key_list"); /* Blackhole is a special case, this allows us to test the upper end @@ -1548,6 +1631,25 @@ drop_schema(MYSQL *mysql, const char *db) DBUG_RETURN(0); } +static int +run_statements(MYSQL *mysql, statement *stmt) +{ + statement *ptr; + DBUG_ENTER("run_statements"); + + for (ptr= stmt; ptr && ptr->length; ptr= ptr->next) + { + if (run_query(mysql, ptr->string, ptr->length)) + { + fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", + my_progname, (uint)ptr->length, ptr->string, mysql_error(mysql)); + exit(1); + } + } + + DBUG_RETURN(0); +} + static int run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) { @@ -1569,7 +1671,7 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) pthread_mutex_unlock(&sleeper_mutex); for (x= 0; x < concur; x++) { - /* nowucreate the thread */ + /* now you create the thread */ if (pthread_create(&mainthread, NULL, run_task, (void *)&con) != 0) { diff --git a/mysql-test/r/mysqlslap.result b/mysql-test/r/mysqlslap.result index 1a8b77fde1c..bca7919d78c 100644 --- a/mysql-test/r/mysqlslap.result +++ b/mysql-test/r/mysqlslap.result @@ -143,3 +143,27 @@ select * from t1; select * from t2; select * from t1; DROP SCHEMA IF EXISTS `mysqlslap`; +DROP SCHEMA IF EXISTS `mysqlslap`; +CREATE SCHEMA `mysqlslap`; +use mysqlslap; +set storage_engine=`heap`; +CREATE TABLE t1 (id int, name varchar(64)); +create table t2(foo1 varchar(32), foo2 varchar(32)); +INSERT INTO t1 VALUES (1, 'This is a test'); +insert into t2 values ('test', 'test2'); +SHOW TABLES; +select * from t1; +SHOW TABLES; +DROP SCHEMA IF EXISTS `mysqlslap`; +DROP SCHEMA IF EXISTS `mysqlslap`; +CREATE SCHEMA `mysqlslap`; +use mysqlslap; +set storage_engine=`myisam`; +CREATE TABLE t1 (id int, name varchar(64)); +create table t2(foo1 varchar(32), foo2 varchar(32)); +INSERT INTO t1 VALUES (1, 'This is a test'); +insert into t2 values ('test', 'test2'); +SHOW TABLES; +select * from t1; +SHOW TABLES; +DROP SCHEMA IF EXISTS `mysqlslap`; diff --git a/mysql-test/t/mysqlslap.test b/mysql-test/t/mysqlslap.test index a40a01f1f25..2a7bbfed932 100644 --- a/mysql-test/t/mysqlslap.test +++ b/mysql-test/t/mysqlslap.test @@ -34,3 +34,5 @@ --exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --auto-generate-sql-load-type=key --auto-generate-sql-execute-number=5 --exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --auto-generate-sql-load-type=key --auto-generate-sql-execute-number=5 --auto-generate-sql-secondary-indexes=3 + +--exec $MYSQL_SLAP --only-print --delimiter=";" --query="select * from t1;select * from t2" --create="CREATE TABLE t1 (id int, name varchar(64)); create table t2(foo1 varchar(32), foo2 varchar(32)); INSERT INTO t1 VALUES (1, 'This is a test'); insert into t2 values ('test', 'test2')" --engine="heap,myisam" --post-query="SHOW TABLES" --pre-query="SHOW TABLES"; From 414da94c045bc3eee1f6ab9d012e731ea32872b6 Mon Sep 17 00:00:00 2001 From: "brian@piggy.tangent.org" <> Date: Mon, 26 Mar 2007 16:45:17 -0700 Subject: [PATCH 20/28] Adjusting locks for concurrency issue (not found, but suspect). --- storage/archive/ha_archive.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/archive/ha_archive.cc b/storage/archive/ha_archive.cc index 72647b58f54..aaafcc86168 100644 --- a/storage/archive/ha_archive.cc +++ b/storage/archive/ha_archive.cc @@ -1003,7 +1003,6 @@ int ha_archive::rnd_init(bool scan) /* We rewind the file so that we can read from the beginning if scan */ if (scan) { - scan_rows= share->rows_recorded; DBUG_PRINT("info", ("archive will retrieve %llu rows", (unsigned long long) scan_rows)); stats.records= 0; @@ -1012,17 +1011,18 @@ int ha_archive::rnd_init(bool scan) If dirty, we lock, and then reset/flush the data. I found that just calling azflush() doesn't always work. */ + pthread_mutex_lock(&share->mutex); + scan_rows= share->rows_recorded; if (share->dirty == TRUE) { - pthread_mutex_lock(&share->mutex); if (share->dirty == TRUE) { DBUG_PRINT("ha_archive", ("archive flushing out rows for scan")); azflush(&(share->archive_write), Z_SYNC_FLUSH); share->dirty= FALSE; } - pthread_mutex_unlock(&share->mutex); } + pthread_mutex_unlock(&share->mutex); if (read_data_header(&archive)) DBUG_RETURN(HA_ERR_CRASHED_ON_USAGE); From 88719428f594723f93745638aaf00ae37727d4fc Mon Sep 17 00:00:00 2001 From: "brian@zim.(none)" <> Date: Mon, 26 Mar 2007 17:15:31 -0700 Subject: [PATCH 21/28] Centralized init logic for starting a scan. --- storage/archive/ha_archive.cc | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/storage/archive/ha_archive.cc b/storage/archive/ha_archive.cc index 72647b58f54..d0f751fce32 100644 --- a/storage/archive/ha_archive.cc +++ b/storage/archive/ha_archive.cc @@ -923,7 +923,7 @@ int ha_archive::index_read(byte *buf, const byte *key, int ha_archive::index_read_idx(byte *buf, uint index, const byte *key, uint key_len, enum ha_rkey_function find_flag) { - int rc= 0; + int rc; bool found= 0; KEY *mkey= &table->s->key_info[index]; current_k_offset= mkey->key_part->offset; @@ -933,22 +933,10 @@ int ha_archive::index_read_idx(byte *buf, uint index, const byte *key, DBUG_ENTER("ha_archive::index_read_idx"); - /* - All of the buffer must be written out or we won't see all of the - data - */ - pthread_mutex_lock(&share->mutex); - azflush(&(share->archive_write), Z_SYNC_FLUSH); - pthread_mutex_unlock(&share->mutex); + rc= rnd_init(TRUE); - /* - Set the position of the local read thread to the beginning postion. - */ - if (read_data_header(&archive)) - { - rc= HA_ERR_CRASHED_ON_USAGE; + if (rc) goto error; - } while (!(get_row(&archive, buf))) { From e661d6eeb66dc8a89229781fe500c62c700b152e Mon Sep 17 00:00:00 2001 From: "brian@zim.(none)" <> Date: Mon, 26 Mar 2007 22:50:08 -0700 Subject: [PATCH 22/28] Making sure thar archive is inited for info AUTO call. --- storage/archive/ha_archive.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/storage/archive/ha_archive.cc b/storage/archive/ha_archive.cc index cacee63277a..6853e879f55 100644 --- a/storage/archive/ha_archive.cc +++ b/storage/archive/ha_archive.cc @@ -1486,6 +1486,7 @@ int ha_archive::info(uint flag) if (flag & HA_STATUS_AUTO) { + init_archive_reader(); azflush(&archive, Z_SYNC_FLUSH); stats.auto_increment_value= archive.auto_increment; } From ef530348cdba1e7319e7203e9850599f85fdba34 Mon Sep 17 00:00:00 2001 From: "kent@mysql.com/kent-amd64.(none)" <> Date: Tue, 27 Mar 2007 17:42:55 +0200 Subject: [PATCH 23/28] mysql.spec.sh, Makefile.am: Don't use explicit calls to mysql-test-run in spec --- Makefile.am | 6 ++++++ support-files/mysql.spec.sh | 5 +---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Makefile.am b/Makefile.am index 70ed8fa7bfe..f620f5d8ab8 100644 --- a/Makefile.am +++ b/Makefile.am @@ -132,6 +132,12 @@ test-force-full: test-force-mem: $(MAKE) 'force=--force --mem' test +test-bt: + -cd mysql-test ; MTR_BUILD_THREAD=auto \ + @PERL@ ./mysql-test-run.pl --force --comment=normal --report-features + -cd mysql-test ; MTR_BUILD_THREAD=auto \ + @PERL@ ./mysql-test-run.pl --force --comment=ps --ps-protocol + # Keep these for a while test-pl: test test-full-pl: test-full diff --git a/support-files/mysql.spec.sh b/support-files/mysql.spec.sh index 15c3e4910d8..ab544b65f74 100644 --- a/support-files/mysql.spec.sh +++ b/support-files/mysql.spec.sh @@ -319,10 +319,7 @@ then cp -fp config.log "$MYSQL_MAXCONFLOG_DEST" fi -( cd mysql-test - perl ./mysql-test-run.pl --force --report-features - perl ./mysql-test-run.pl --force --ps-protocol - true ) +make test-bt # Save mysqld-max ./libtool --mode=execute cp sql/mysqld sql/mysqld-max From e5b660487bf584504fab2e6a0e114a3919fe34a0 Mon Sep 17 00:00:00 2001 From: "serg@janus.mylan" <> Date: Wed, 28 Mar 2007 15:33:29 +0200 Subject: [PATCH 24/28] restored run-time thread lib detection --- include/my_pthread.h | 9 +++++++++ include/thr_alarm.h | 8 +++----- mysys/my_pthread.c | 10 +++++----- mysys/my_thr_init.c | 21 +++++++++++++++++++++ mysys/thr_alarm.c | 44 ++++++++++++++++++++++---------------------- sql/mysqld.cc | 34 +++++++++++++++++++--------------- sql/stacktrace.c | 17 ++--------------- sql/stacktrace.h | 2 -- 8 files changed, 81 insertions(+), 64 deletions(-) diff --git a/include/my_pthread.h b/include/my_pthread.h index 4df105e1b20..340dc32981a 100644 --- a/include/my_pthread.h +++ b/include/my_pthread.h @@ -701,6 +701,15 @@ extern uint my_thread_end_wait_time; Keep track of shutdown,signal, and main threads so that my_end() will not report errors with them */ + +/* Which kind of thread library is in use */ + +#define THD_LIB_OTHER 1 +#define THD_LIB_NPTL 2 +#define THD_LIB_LT 4 + +extern uint thd_lib_detected; + /* statistics_xxx functions are for not essential statistic */ #ifndef thread_safe_increment diff --git a/include/thr_alarm.h b/include/thr_alarm.h index 14dd538c9e4..a2694ba105b 100644 --- a/include/thr_alarm.h +++ b/include/thr_alarm.h @@ -24,11 +24,6 @@ extern "C" { #ifndef USE_ALARM_THREAD #define USE_ONE_SIGNAL_HAND /* One must call process_alarm */ #endif -#ifdef HAVE_LINUXTHREADS -#define THR_CLIENT_ALARM SIGALRM -#else -#define THR_CLIENT_ALARM SIGUSR1 -#endif #ifdef HAVE_rts_threads #undef USE_ONE_SIGNAL_HAND #define USE_ALARM_THREAD @@ -90,6 +85,9 @@ typedef struct st_alarm { my_bool malloced; } ALARM; +extern uint thr_client_alarm; +extern pthread_t alarm_thread; + #define thr_alarm_init(A) (*(A))=0 #define thr_alarm_in_use(A) (*(A)!= 0) void init_thr_alarm(uint max_alarm); diff --git a/mysys/my_pthread.c b/mysys/my_pthread.c index 74ba98c321a..db01602f4ab 100644 --- a/mysys/my_pthread.c +++ b/mysys/my_pthread.c @@ -31,6 +31,8 @@ uint thd_lib_detected= 0; +uint thd_lib_detected; + #ifndef my_pthread_setprio void my_pthread_setprio(pthread_t thread_id,int prior) { @@ -51,8 +53,6 @@ int my_pthread_getprio(pthread_t thread_id) int policy; if (!pthread_getschedparam(thread_id,&policy,&tmp_sched_param)) { - DBUG_PRINT("thread",("policy: %d priority: %d", - policy,tmp_sched_param.sched_priority)); return tmp_sched_param.sched_priority; } #endif @@ -314,8 +314,6 @@ void sigwait_handle_sig(int sig) pthread_mutex_unlock(&LOCK_sigwait); } -extern pthread_t alarm_thread; - void *sigwait_thread(void *set_arg) { sigset_t *set=(sigset_t*) set_arg; @@ -334,7 +332,9 @@ void *sigwait_thread(void *set_arg) sigaction(i, &sact, (struct sigaction*) 0); } } - sigaddset(set,THR_CLIENT_ALARM); + /* Ensure that init_thr_alarm() is called */ + DBUG_ASSERT(thr_client_alarm); + sigaddset(set, thr_client_alarm); pthread_sigmask(SIG_UNBLOCK,(sigset_t*) set,(sigset_t*) 0); alarm_thread=pthread_self(); /* For thr_alarm */ diff --git a/mysys/my_thr_init.c b/mysys/my_thr_init.c index 61e6e640027..464edb77f99 100644 --- a/mysys/my_thr_init.c +++ b/mysys/my_thr_init.c @@ -20,6 +20,7 @@ #include "mysys_priv.h" #include +#include #ifdef THREAD #ifdef USE_TLS @@ -63,6 +64,8 @@ nptl_pthread_exit_hack_handler(void *arg __attribute((unused))) #endif +static uint get_thread_lib(void); + /* initialize thread environment @@ -76,6 +79,8 @@ nptl_pthread_exit_hack_handler(void *arg __attribute((unused))) my_bool my_thread_global_init(void) { + thd_lib_detected= get_thread_lib(); + if (pthread_key_create(&THR_KEY_mysys,0)) { fprintf(stderr,"Can't initialize threads: error %d\n",errno); @@ -395,4 +400,20 @@ const char *my_thread_name(void) } #endif /* DBUG_OFF */ + +static uint get_thread_lib(void) +{ + char buff[64]; + +#ifdef _CS_GNU_LIBPTHREAD_VERSION + confstr(_CS_GNU_LIBPTHREAD_VERSION, buff, sizeof(buff)); + + if (!strncasecmp(buff, "NPTL", 4)) + return THD_LIB_NPTL; + if (!strncasecmp(buff, "linuxthreads", 12)) + return THD_LIB_LT; +#endif + return THD_LIB_OTHER; +} + #endif /* THREAD */ diff --git a/mysys/thr_alarm.c b/mysys/thr_alarm.c index 3fd70790281..57670c9ac14 100644 --- a/mysys/thr_alarm.c +++ b/mysys/thr_alarm.c @@ -34,6 +34,7 @@ #define ETIME ETIMEDOUT #endif +uint thr_client_alarm; static int alarm_aborted=1; /* No alarm thread */ my_bool thr_alarm_inited= 0; volatile my_bool alarm_thread_running= 0; @@ -56,9 +57,7 @@ static void *alarm_handler(void *arg); #define reschedule_alarms() pthread_kill(alarm_thread,THR_SERVER_ALARM) #endif -#if THR_CLIENT_ALARM != SIGALRM || defined(USE_ALARM_THREAD) static sig_handler thread_alarm(int sig __attribute__((unused))); -#endif static int compare_ulong(void *not_used __attribute__((unused)), byte *a_ptr,byte* b_ptr) @@ -77,9 +76,13 @@ void init_thr_alarm(uint max_alarms) sigfillset(&full_signal_set); /* Neaded to block signals */ pthread_mutex_init(&LOCK_alarm,MY_MUTEX_INIT_FAST); pthread_cond_init(&COND_alarm,NULL); -#if THR_CLIENT_ALARM != SIGALRM || defined(USE_ALARM_THREAD) - my_sigset(THR_CLIENT_ALARM,thread_alarm); + thr_client_alarm= thd_lib_detected == THD_LIB_LT ? SIGALRM : SIGUSR1; +#ifndef USE_ALARM_THREAD + if (thd_lib_detected != THD_LIB_LT) #endif + { + my_sigset(thr_client_alarm, thread_alarm); + } sigemptyset(&s); sigaddset(&s, THR_SERVER_ALARM); alarm_thread=pthread_self(); @@ -97,10 +100,11 @@ void init_thr_alarm(uint max_alarms) } #elif defined(USE_ONE_SIGNAL_HAND) pthread_sigmask(SIG_BLOCK, &s, NULL); /* used with sigwait() */ -#if THR_SERVER_ALARM == THR_CLIENT_ALARM - my_sigset(THR_CLIENT_ALARM,process_alarm); /* Linuxthreads */ - pthread_sigmask(SIG_UNBLOCK, &s, NULL); -#endif + if (thd_lib_detected == THD_LIB_LT) + { + my_sigset(thr_client_alarm, process_alarm); /* Linuxthreads */ + pthread_sigmask(SIG_UNBLOCK, &s, NULL); + } #else my_sigset(THR_SERVER_ALARM, process_alarm); pthread_sigmask(SIG_UNBLOCK, &s, NULL); @@ -152,7 +156,7 @@ my_bool thr_alarm(thr_alarm_t *alrm, uint sec, ALARM *alarm_data) now=(ulong) time((time_t*) 0); pthread_sigmask(SIG_BLOCK,&full_signal_set,&old_mask); - pthread_mutex_lock(&LOCK_alarm); /* Lock from threads & alarms */ + pthread_mutex_lock(&LOCK_alarm); /* Lock from threads & alarms */ if (alarm_aborted > 0) { /* No signal thread */ DBUG_PRINT("info", ("alarm aborted")); @@ -273,18 +277,17 @@ sig_handler process_alarm(int sig __attribute__((unused))) This must be first as we can't call DBUG inside an alarm for a normal thread */ -#if THR_SERVER_ALARM == THR_CLIENT_ALARM - if (!pthread_equal(pthread_self(),alarm_thread)) + if (thd_lib_detected == THD_LIB_LT && + !pthread_equal(pthread_self(),alarm_thread)) { #if defined(MAIN) && !defined(__bsdi__) printf("thread_alarm in process_alarm\n"); fflush(stdout); #endif #ifdef DONT_REMEMBER_SIGNAL - my_sigset(THR_CLIENT_ALARM,process_alarm); /* int. thread system calls */ + my_sigset(thr_client_alarm, process_alarm); /* int. thread system calls */ #endif return; } -#endif /* We have to do do the handling of the alarm in a sub function, @@ -328,7 +331,7 @@ static sig_handler process_alarm_part2(int sig __attribute__((unused))) alarm_data=(ALARM*) queue_element(&alarm_queue,i); alarm_data->alarmed=1; /* Info to thread */ if (pthread_equal(alarm_data->thread,alarm_thread) || - pthread_kill(alarm_data->thread, THR_CLIENT_ALARM)) + pthread_kill(alarm_data->thread, thr_client_alarm)) { #ifdef MAIN printf("Warning: pthread_kill couldn't find thread!!!\n"); @@ -352,7 +355,7 @@ static sig_handler process_alarm_part2(int sig __attribute__((unused))) alarm_data->alarmed=1; /* Info to thread */ DBUG_PRINT("info",("sending signal to waiting thread")); if (pthread_equal(alarm_data->thread,alarm_thread) || - pthread_kill(alarm_data->thread, THR_CLIENT_ALARM)) + pthread_kill(alarm_data->thread, thr_client_alarm)) { #ifdef MAIN printf("Warning: pthread_kill couldn't find thread!!!\n"); @@ -488,7 +491,7 @@ void thr_alarm_info(ALARM_INFO *info) ARGSUSED */ -#if THR_CLIENT_ALARM != SIGALRM || defined(USE_ALARM_THREAD) + static sig_handler thread_alarm(int sig) { #ifdef MAIN @@ -498,7 +501,6 @@ static sig_handler thread_alarm(int sig) my_sigset(sig,thread_alarm); /* int. thread system calls */ #endif } -#endif #ifdef HAVE_TIMESPEC_TS_SEC @@ -784,9 +786,7 @@ static void *signal_hand(void *arg __attribute__((unused))) sigaddset(&set,SIGINT); sigaddset(&set,SIGQUIT); sigaddset(&set,SIGTERM); -#if THR_CLIENT_ALARM != SIGHUP sigaddset(&set,SIGHUP); -#endif #ifdef SIGTSTP sigaddset(&set,SIGTSTP); #endif @@ -797,7 +797,7 @@ static void *signal_hand(void *arg __attribute__((unused))) puts("Starting signal handling thread"); #endif printf("server alarm: %d thread alarm: %d\n", - THR_SERVER_ALARM,THR_CLIENT_ALARM); + THR_SERVER_ALARM, thr_client_alarm); DBUG_PRINT("info",("Starting signal and alarm handling thread")); for(;;) { @@ -865,11 +865,11 @@ int main(int argc __attribute__((unused)),char **argv __attribute__((unused))) sigaddset(&set,SIGTSTP); #endif sigaddset(&set,THR_SERVER_ALARM); - sigdelset(&set,THR_CLIENT_ALARM); + sigdelset(&set, thr_client_alarm); (void) pthread_sigmask(SIG_SETMASK,&set,NULL); #ifdef NOT_USED sigemptyset(&set); - sigaddset(&set,THR_CLIENT_ALARM); + sigaddset(&set, thr_client_alarm); VOID(pthread_sigmask(SIG_UNBLOCK, &set, (sigset_t*) 0)); #endif diff --git a/sql/mysqld.cc b/sql/mysqld.cc index a0687929de5..c5933484675 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -195,12 +195,6 @@ inline void reset_floating_point_exceptions() } /* cplusplus */ - -#if defined(HAVE_LINUXTHREADS) -#define THR_KILL_SIGNAL SIGINT -#else -#define THR_KILL_SIGNAL SIGUSR2 // Can't use this with LinuxThreads -#endif #define MYSQL_KILL_SIGNAL SIGTERM #ifdef HAVE_GLIBC2_STYLE_GETHOSTBYNAME_R @@ -605,6 +599,7 @@ pthread_mutex_t LOCK_server_started; pthread_cond_t COND_server_started; int mysqld_server_started= 0; +static uint thr_kill_signal; File_parser_dummy_hook file_parser_dummy_hook; @@ -788,7 +783,7 @@ static void close_connections(void) DBUG_PRINT("info",("Waiting for select thread")); #ifndef DONT_USE_THR_ALARM - if (pthread_kill(select_thread,THR_CLIENT_ALARM)) + if (pthread_kill(select_thread, thr_client_alarm)) break; // allready dead #endif set_timespec(abstime, 2); @@ -2294,7 +2289,9 @@ static void init_signals(void) DBUG_ENTER("init_signals"); if (test_flags & TEST_SIGINT) - my_sigset(THR_KILL_SIGNAL,end_thread_signal); + { + my_sigset(thr_kill_signal, end_thread_signal); + } my_sigset(THR_SERVER_ALARM,print_signal_warning); // Should never be called! if (!(test_flags & TEST_NO_STACKTRACE) || (test_flags & TEST_CORE_ON_SIGNAL)) @@ -2351,8 +2348,12 @@ static void init_signals(void) #endif sigaddset(&set,THR_SERVER_ALARM); if (test_flags & TEST_SIGINT) - sigdelset(&set,THR_KILL_SIGNAL); // May be SIGINT - sigdelset(&set,THR_CLIENT_ALARM); // For alarms + { + // May be SIGINT + sigdelset(&set, thr_kill_signal); + } + // For alarms + sigdelset(&set, thr_client_alarm); sigprocmask(SIG_SETMASK,&set,NULL); pthread_sigmask(SIG_SETMASK,&set,NULL); DBUG_VOID_RETURN; @@ -2415,23 +2416,19 @@ pthread_handler_t signal_hand(void *arg __attribute__((unused))) */ init_thr_alarm(thread_scheduler.max_threads + global_system_variables.max_insert_delayed_threads + 10); -#if SIGINT != THR_KILL_SIGNAL - if (test_flags & TEST_SIGINT) + if (thd_lib_detected != THD_LIB_LT && (test_flags & TEST_SIGINT)) { (void) sigemptyset(&set); // Setup up SIGINT for debug (void) sigaddset(&set,SIGINT); // For debugging (void) pthread_sigmask(SIG_UNBLOCK,&set,NULL); } -#endif (void) sigemptyset(&set); // Setup up SIGINT for debug #ifdef USE_ONE_SIGNAL_HAND (void) sigaddset(&set,THR_SERVER_ALARM); // For alarms #endif #ifndef IGNORE_SIGHUP_SIGQUIT (void) sigaddset(&set,SIGQUIT); -#if THR_CLIENT_ALARM != SIGHUP (void) sigaddset(&set,SIGHUP); -#endif #endif (void) sigaddset(&set,SIGTERM); (void) sigaddset(&set,SIGTSTP); @@ -3626,6 +3623,13 @@ int main(int argc, char **argv) MY_INIT(argv[0]); // init my_sys library & pthreads /* nothing should come before this line ^^^ */ + /* Set signal used to kill MySQL */ +#if defined(SIGUSR2) + thr_kill_signal= thd_lib_detected == THD_LIB_LT ? SIGINT : SIGUSR2; +#else + thr_kill_signal= SIGINT; +#endif + /* Perform basic logger initialization logger. Should be called after MY_INIT, as it initializes mutexes. Log tables are inited later. diff --git a/sql/stacktrace.c b/sql/stacktrace.c index d8e9b7fd883..0dbf4522a71 100644 --- a/sql/stacktrace.c +++ b/sql/stacktrace.c @@ -55,19 +55,6 @@ void safe_print_str(const char* name, const char* val, int max_len) static my_bool is_nptl; -/* Check if we are using NPTL or LinuxThreads on Linux */ -void check_thread_lib(void) -{ - char buf[5]; - -#ifdef _CS_GNU_LIBPTHREAD_VERSION - confstr(_CS_GNU_LIBPTHREAD_VERSION, buf, sizeof(buf)); - is_nptl = !strncasecmp(buf, "NPTL", sizeof(buf)); -#else - is_nptl = 0; -#endif -} - #if defined(__alpha__) && defined(__GNUC__) /* The only way to backtrace without a symbol table on alpha @@ -173,8 +160,8 @@ terribly wrong...\n"); #endif /* __alpha__ */ /* We are 1 frame above signal frame with NPTL and 2 frames above with LT */ - sigreturn_frame_count = is_nptl ? 1 : 2; - + sigreturn_frame_count = thd_lib_detected == THD_LIB_LT ? 2 : 1; + while (fp < (uchar**) stack_bottom) { #if defined(__i386__) || defined(__x86_64__) diff --git a/sql/stacktrace.h b/sql/stacktrace.h index f5c92e54e1c..56b9877180a 100644 --- a/sql/stacktrace.h +++ b/sql/stacktrace.h @@ -27,11 +27,9 @@ extern char* heap_start; #define init_stacktrace() do { \ heap_start = (char*) &__bss_start; \ - check_thread_lib(); \ } while(0); void print_stacktrace(gptr stack_bottom, ulong thread_stack); void safe_print_str(const char* name, const char* val, int max_len); -void check_thread_lib(void); #endif /* (defined (__i386__) || (defined(__alpha__) && defined(__GNUC__))) */ #endif /* TARGET_OS_LINUX */ From fd1d4f864ee3de2fd317e529fceebcecd73cb63f Mon Sep 17 00:00:00 2001 From: "serg@janus.mylan" <> Date: Wed, 28 Mar 2007 17:12:38 +0200 Subject: [PATCH 25/28] compiler warnings --- sql/mysqld.cc | 2 +- sql/stacktrace.c | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index ce53cd22669..51feb197713 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -599,7 +599,6 @@ pthread_mutex_t LOCK_server_started; pthread_cond_t COND_server_started; int mysqld_server_started= 0; -static uint thr_kill_signal; File_parser_dummy_hook file_parser_dummy_hook; @@ -633,6 +632,7 @@ struct rand_struct sql_rand; // used by sql_class.cc:THD::THD() #ifndef EMBEDDED_LIBRARY struct passwd *user_info; static pthread_t select_thread; +static uint thr_kill_signal; #endif /* OS specific variables */ diff --git a/sql/stacktrace.c b/sql/stacktrace.c index 0dbf4522a71..078f62c6b2b 100644 --- a/sql/stacktrace.c +++ b/sql/stacktrace.c @@ -53,8 +53,6 @@ void safe_print_str(const char* name, const char* val, int max_len) #define SIGRETURN_FRAME_OFFSET 23 #endif -static my_bool is_nptl; - #if defined(__alpha__) && defined(__GNUC__) /* The only way to backtrace without a symbol table on alpha From 385526f768d5a335156f5968afcaa7447a1529f8 Mon Sep 17 00:00:00 2001 From: "joerg@trift-lap.fambruehe" <> Date: Wed, 28 Mar 2007 19:01:49 +0200 Subject: [PATCH 26/28] mysys/my_thr_init.c : Avoid warnings of "unused variable" by extending a '#ifdef'. --- mysys/my_thr_init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysys/my_thr_init.c b/mysys/my_thr_init.c index 464edb77f99..75da07390ba 100644 --- a/mysys/my_thr_init.c +++ b/mysys/my_thr_init.c @@ -403,9 +403,9 @@ const char *my_thread_name(void) static uint get_thread_lib(void) { +#ifdef _CS_GNU_LIBPTHREAD_VERSION char buff[64]; -#ifdef _CS_GNU_LIBPTHREAD_VERSION confstr(_CS_GNU_LIBPTHREAD_VERSION, buff, sizeof(buff)); if (!strncasecmp(buff, "NPTL", 4)) From 7a2e0ee7d757d71df26c20ab23b11b61c82fd5c7 Mon Sep 17 00:00:00 2001 From: "kent@mysql.com/kent-amd64.(none)" <> Date: Wed, 28 Mar 2007 23:00:50 +0200 Subject: [PATCH 27/28] configure.in: Don't install ndb man pages if no ndb configured config-win.h, CMakeLists.txt, README, configure.js: Removed Cybozu patches --- CMakeLists.txt | 3 --- configure.in | 12 +++++++++--- include/config-win.h | 14 -------------- win/README | 1 - win/configure.js | 1 - 5 files changed, 9 insertions(+), 22 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a76361693a..3a286071bb0 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,9 +80,6 @@ CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/support-files/my-small.cnf.sh IF(__NT__) ADD_DEFINITIONS(-D __NT__) ENDIF(__NT__) -IF(CYBOZU) - ADD_DEFINITIONS(-D CYBOZU) -ENDIF(CYBOZU) # in some places we use DBUG_OFF SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -D DBUG_OFF") diff --git a/configure.in b/configure.in index 7867bf444ad..9b0505a594f 100644 --- a/configure.in +++ b/configure.in @@ -2376,12 +2376,18 @@ AC_ARG_WITH(man, [with_man=yes] ) -if test "$with_man" = "yes" +if test X"$with_man" = Xyes then man_dirs="man" - man1_files=`ls -1 $srcdir/man/*.1 | sed -e 's;^.*man/;;'` + if test X"$have_ndbcluster" = Xyes + then + man1_files=`ls $srcdir/man/*.1 | sed -e 's;^.*man/;;'` + man8_files=`ls $srcdir/man/*.8 | sed -e 's;^.*man/;;'` + else + man1_files=`ls $srcdir/man/*.1 | grep -v '/ndb' | sed -e 's;^.*man/;;'` + man8_files=`ls $srcdir/man/*.8 | grep -v '/ndb' | sed -e 's;^.*man/;;'` + fi man1_files=`echo $man1_files` - man8_files=`ls -1 $srcdir/man/*.8 | sed -e 's;^.*man/;;'` man8_files=`echo $man8_files` else man_dirs="" diff --git a/include/config-win.h b/include/config-win.h index 34e828e01e8..279be7aa5e4 100644 --- a/include/config-win.h +++ b/include/config-win.h @@ -408,14 +408,8 @@ inline double ulonglong2double(ulonglong value) #define shared_memory_buffer_length 16000 #define default_shared_memory_base_name "MYSQL" -#ifdef CYBOZU -#define MYSQL_DEFAULT_CHARSET_NAME "utf8" -#define MYSQL_DEFAULT_COLLATION_NAME "utf8_general_cs" -#define HAVE_UTF8_GENERAL_CS 1 -#else #define MYSQL_DEFAULT_CHARSET_NAME "latin1" #define MYSQL_DEFAULT_COLLATION_NAME "latin1_swedish_ci" -#endif #define HAVE_SPATIAL 1 #define HAVE_RTREE_KEYS 1 @@ -426,10 +420,8 @@ inline double ulonglong2double(ulonglong value) /* Define charsets you want */ /* #undef HAVE_CHARSET_armscii8 */ /* #undef HAVE_CHARSET_ascii */ -#ifndef CYBOZU #define HAVE_CHARSET_big5 1 #define HAVE_CHARSET_cp1250 1 -#endif /* #undef HAVE_CHARSET_cp1251 */ /* #undef HAVE_CHARSET_cp1256 */ /* #undef HAVE_CHARSET_cp1257 */ @@ -438,33 +430,27 @@ inline double ulonglong2double(ulonglong value) /* #undef HAVE_CHARSET_cp866 */ #define HAVE_CHARSET_cp932 1 /* #undef HAVE_CHARSET_dec8 */ -#ifndef CYBOZU #define HAVE_CHARSET_eucjpms 1 #define HAVE_CHARSET_euckr 1 #define HAVE_CHARSET_gb2312 1 #define HAVE_CHARSET_gbk 1 -#endif /* #undef HAVE_CHARSET_greek */ /* #undef HAVE_CHARSET_hebrew */ /* #undef HAVE_CHARSET_hp8 */ /* #undef HAVE_CHARSET_keybcs2 */ /* #undef HAVE_CHARSET_koi8r */ /* #undef HAVE_CHARSET_koi8u */ -#ifndef CYBOZU #define HAVE_CHARSET_latin1 1 #define HAVE_CHARSET_latin2 1 -#endif /* #undef HAVE_CHARSET_latin5 */ /* #undef HAVE_CHARSET_latin7 */ /* #undef HAVE_CHARSET_macce */ /* #undef HAVE_CHARSET_macroman */ #define HAVE_CHARSET_sjis 1 /* #undef HAVE_CHARSET_swe7 */ -#ifndef CYBOZU #define HAVE_CHARSET_tis620 1 #define HAVE_CHARSET_ucs2 1 #define HAVE_CHARSET_ujis 1 -#endif #define HAVE_CHARSET_utf8 1 #define HAVE_UCA_COLLATIONS 1 diff --git a/win/README b/win/README index 8fe3dc21d1e..871ae4efee7 100644 --- a/win/README +++ b/win/README @@ -48,7 +48,6 @@ The options right now are MYSQL_SERVER_SUFFIX= Server suffix, default none COMPILATION_COMMENT= Server comment, default "Source distribution" MYSQL_TCP_PORT= Server port, default 3306 - CYBOZU DISABLE_GRANT_OPTIONS Disables the use of --init-file and --skip-grant-tables options of mysqld.exe diff --git a/win/configure.js b/win/configure.js index 59c73fc2fab..3488efacba3 100755 --- a/win/configure.js +++ b/win/configure.js @@ -46,7 +46,6 @@ try case "WITH_INNOBASE_STORAGE_ENGINE": case "WITH_PARTITION_STORAGE_ENGINE": case "__NT__": - case "CYBOZU": case "DISABLE_GRANT_OPTIONS": configfile.WriteLine("SET (" + args.Item(i) + " TRUE)"); break; From 7eeae6ff13a5e017af43d4474676958ec5c837cc Mon Sep 17 00:00:00 2001 From: "kent@mysql.com/kent-amd64.(none)" <> Date: Wed, 28 Mar 2007 23:18:43 +0200 Subject: [PATCH 28/28] Makefile.am: Extended test section Removed duplicate EXTRA_DIST --- Makefile.am | 18 +++++++++++++++--- client/Makefile.am | 3 +-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Makefile.am b/Makefile.am index d742539da26..c4a4a040a40 100644 --- a/Makefile.am +++ b/Makefile.am @@ -149,13 +149,25 @@ test-force-mem: test-bt: -cd mysql-test ; MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --force --comment=normal --report-features + @PERL@ ./mysql-test-run.pl --comment=normal --force --timer \ + --skip-ndbcluster --report-features -cd mysql-test ; MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --force --comment=ps --ps-protocol + @PERL@ ./mysql-test-run.pl --comment=ps --force --timer \ + --skip-ndbcluster --ps-protocol + -cd mysql-test ; MTR_BUILD_THREAD=auto \ + @PERL@ ./mysql-test-run.pl --comment=normal+rowrepl --force --timer \ + --skip-ndbcluster --mysqld=--binlog-format=row + -cd mysql-test ; MTR_BUILD_THREAD=auto \ + @PERL@ ./mysql-test-run.pl --comment=ps+rowrepl+NDB --force --timer \ + --ps-protocol --mysqld=--binlog-format=row + -cd mysql-test ; MTR_BUILD_THREAD=auto \ + @PERL@ ./mysql-test-run.pl --comment=NDB --force --timer \ + --with-ndbcluster-only test-bt-debug: -cd mysql-test ; MTR_BUILD_THREAD=auto \ - @PERL@ ./mysql-test-run.pl --force --comment=debug --report-features + @PERL@ ./mysql-test-run.pl --comment=debug --force --timer \ + --skip-ndbcluster --skip-rpl --report-features # Keep these for a while test-pl: test diff --git a/client/Makefile.am b/client/Makefile.am index 1c908ea715e..5960095778b 100644 --- a/client/Makefile.am +++ b/client/Makefile.am @@ -34,7 +34,7 @@ LDADD= @CLIENT_EXTRA_LDFLAGS@ $(CLIENT_THREAD_LIBS) \ noinst_HEADERS = sql_string.h completion_hash.h my_readline.h \ client_priv.h -EXTRA_DIST = get_password.c CMakeLists.txt +EXTRA_DIST = get_password.c CMakeLists.txt echo.c bin_PROGRAMS = mysql \ mysqladmin \ @@ -96,7 +96,6 @@ DEFS = -DUNDEF_THREADS_HACK \ sql_src=log_event.h mysql_priv.h log_event.cc my_decimal.h my_decimal.cc strings_src=decimal.c -EXTRA_DIST = get_password.c CMakeLists.txt echo.c link_sources: for f in $(sql_src) ; do \