From 7ab5c6820c1a58f66b612d76373e539485bc3966 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 6 Mar 2007 17:14:59 -0800 Subject: [PATCH 01/11] Fixed buffer overflow cases (should not be possible to do...) Fixed for autoincrement support/GUID client/client_priv.h: Option for autoincrement client/mysqlslap.c: Fixed buffer overflow where sprintf is used (now removed). Fixed for auto-increment support/GUID. mysql-test/t/mysqlslap.test: Added test for auto increment, update, write, read, and mixed. --- client/client_priv.h | 2 + client/mysqlslap.c | 373 +++++++++++++++++++++++++++++++++--- mysql-test/t/mysqlslap.test | 14 ++ 3 files changed, 359 insertions(+), 30 deletions(-) diff --git a/client/client_priv.h b/client/client_priv.h index 646619f62b1..c8cf68bfd13 100644 --- a/client/client_priv.h +++ b/client/client_priv.h @@ -58,6 +58,8 @@ enum options_client OPT_TZ_UTC, OPT_AUTO_CLOSE, OPT_CREATE_SLAP_SCHEMA, OPT_SLAP_CSV, OPT_SLAP_CREATE_STRING, OPT_SLAP_AUTO_GENERATE_SQL_LOAD_TYPE, OPT_SLAP_AUTO_GENERATE_WRITE_NUM, + OPT_SLAP_AUTO_GENERATE_ADD_AUTO, + OPT_SLAP_AUTO_GENERATE_GUID_PRIMARY, OPT_MYSQL_REPLACE_INTO, OPT_BASE64_OUTPUT, OPT_SERVER_ID, OPT_FIX_TABLE_NAMES, OPT_FIX_DB_NAMES, OPT_SSL_VERIFY_SERVER_CERT, OPT_DEBUG_INFO, OPT_COLUMN_TYPES diff --git a/client/mysqlslap.c b/client/mysqlslap.c index ad2c8685ba1..047193495d2 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -71,9 +71,16 @@ TODO: #define SHOW_VERSION "0.9" -#define HUGE_STRING_LENGTH 8096 +#define HUGE_STRING_LENGTH 8196 #define RAND_STRING_SIZE 126 +/* Types */ +#define SELECT_TYPE 0 +#define UPDATE_TYPE 1 +#define INSERT_TYPE 2 +#define UPDATE_TYPE_REQUERIES_PREFIX 3 +#define CREATE_TABLE_TYPE 4 + #include "client_priv.h" #ifdef HAVE_LIBPTHREAD #include @@ -107,6 +114,9 @@ static char *shared_memory_base_name=0; static char **defaults_argv; +char *primary_keys; +unsigned long long primary_keys_number_of; + static char *host= NULL, *opt_password= NULL, *user= NULL, *user_supplied_query= NULL, *default_engine= NULL, @@ -127,6 +137,8 @@ static my_bool opt_slave; static my_bool opt_compress= FALSE, tty_password= FALSE, opt_silent= FALSE, + auto_generate_sql_autoincrement= FALSE, + auto_generate_sql_guid_primary= FALSE, auto_generate_sql= FALSE; const char *auto_generate_sql_type= "mixed"; @@ -159,6 +171,7 @@ typedef struct statement statement; struct statement { char *string; size_t length; + unsigned char type; statement *next; }; @@ -206,7 +219,10 @@ static int drop_schema(MYSQL *mysql, const char *db); uint get_random_string(char *buf); static statement *build_table_string(void); static statement *build_insert_string(void); +static statement *build_update_string(void); static statement *build_query_string(void); +static int generate_primary_key_list(MYSQL *mysql); +static int drop_primary_key_list(void); static int create_schema(MYSQL *mysql, const char *db, statement *stmt, statement *engine_stmt); static int run_scheduler(stats *sptr, statement *stmts, uint concur, @@ -343,6 +359,13 @@ int main(int argc, char **argv) /* First we create */ if (create_statements) create_schema(&mysql, create_schema_string, create_statements, eptr); + + /* + If we generated GUID we need to build a list of them from creation that + we can later use. + */ + if (auto_generate_sql_guid_primary) + generate_primary_key_list(&mysql); run_scheduler(sptr, query_statements, *current, client_limit); } @@ -359,6 +382,10 @@ int main(int argc, char **argv) if (!opt_preserve) drop_schema(&mysql, create_schema_string); + + if (auto_generate_sql_guid_primary) + drop_primary_key_list(); + } while (eptr ? (eptr= eptr->next) : 0); if (!opt_only_print) @@ -397,8 +424,18 @@ static struct my_option my_long_options[] = "Generate SQL where not supplied by file or command line.", (gptr*) &auto_generate_sql, (gptr*) &auto_generate_sql, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"auto-generate-sql-add-autoincrement", OPT_SLAP_AUTO_GENERATE_ADD_AUTO, + "Add autoincrement to auto-generated tables.", + (gptr*) &auto_generate_sql_autoincrement, + (gptr*) &auto_generate_sql_autoincrement, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"auto-generate-sql-guid-primary", OPT_SLAP_AUTO_GENERATE_GUID_PRIMARY, + "Add GUID based primary keys to auto-generated tables.", + (gptr*) &auto_generate_sql_guid_primary, + (gptr*) &auto_generate_sql_guid_primary, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"auto-generate-sql-load-type", OPT_SLAP_AUTO_GENERATE_SQL_LOAD_TYPE, - "Load types are mixed, write, or read. Default is mixed\n", + "Load types are mixed, update, write, or read. Default is mixed\n", (gptr*) &auto_generate_sql_type, (gptr*) &auto_generate_sql_type, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"auto-generate-sql-write-number", OPT_SLAP_AUTO_GENERATE_WRITE_NUM, @@ -614,7 +651,7 @@ get_random_string(char *buf) static statement * build_table_string(void) { - char buf[512]; + char buf[HUGE_STRING_LENGTH]; int col_count; statement *ptr; DYNAMIC_STRING table_string; @@ -626,22 +663,63 @@ build_table_string(void) init_dynamic_string(&table_string, "", 1024, 1024); dynstr_append(&table_string, "CREATE TABLE `t1` ("); - for (col_count= 1; col_count <= num_int_cols; col_count++) + + if (auto_generate_sql_autoincrement) { - sprintf(buf, "intcol%d INT(32)", col_count); + if (snprintf(buf, HUGE_STRING_LENGTH, "id serial") > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in create table\n"); + exit(1); + } dynstr_append(&table_string, buf); - if (col_count < num_int_cols || num_char_cols > 0) + if (num_int_cols || num_char_cols) dynstr_append(&table_string, ","); } - for (col_count= 1; col_count <= num_char_cols; col_count++) + + if (auto_generate_sql_guid_primary) { - sprintf(buf, "charcol%d VARCHAR(128)", col_count); + if (snprintf(buf, HUGE_STRING_LENGTH, "id varchar(32) primary key") > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in create table\n"); + exit(1); + } dynstr_append(&table_string, buf); - if (col_count < num_char_cols) + if (num_int_cols || num_char_cols) dynstr_append(&table_string, ","); } + + if (num_int_cols) + for (col_count= 1; col_count <= num_int_cols; col_count++) + { + if (snprintf(buf, HUGE_STRING_LENGTH, "intcol%d INT(32)", col_count) + > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in create table\n"); + exit(1); + } + dynstr_append(&table_string, buf); + + if (col_count < num_int_cols || num_char_cols > 0) + dynstr_append(&table_string, ","); + } + + if (num_char_cols) + for (col_count= 1; col_count <= num_char_cols; col_count++) + { + if (snprintf(buf, HUGE_STRING_LENGTH, "charcol%d VARCHAR(128)", col_count) + > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating table\n"); + exit(1); + } + dynstr_append(&table_string, buf); + + if (col_count < num_char_cols) + dynstr_append(&table_string, ","); + } + dynstr_append(&table_string, ")"); ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)); ptr->string = (char *)my_malloc(table_string.length+1, MYF(MY_WME)); @@ -651,6 +729,94 @@ build_table_string(void) DBUG_RETURN(ptr); } +/* + build_update_string() + + This function builds insert statements when the user opts to not supply + an insert file or string containing insert data +*/ +static statement * +build_update_string(void) +{ + char buf[HUGE_STRING_LENGTH]; + int col_count; + statement *ptr; + DYNAMIC_STRING update_string; + DBUG_ENTER("build_update_string"); + + init_dynamic_string(&update_string, "", 1024, 1024); + + dynstr_append(&update_string, "UPDATE t1 SET "); + + if (num_int_cols) + for (col_count= 1; col_count <= num_int_cols; col_count++) + { + if (snprintf(buf, HUGE_STRING_LENGTH, "intcol%d = %ld", col_count, + random()) > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating update\n"); + exit(1); + } + dynstr_append(&update_string, buf); + + if (col_count < num_int_cols || num_char_cols > 0) + dynstr_append_mem(&update_string, ",", 1); + } + + if (num_char_cols) + for (col_count= 1; col_count <= num_char_cols; col_count++) + { + char rand_buffer[RAND_STRING_SIZE]; + int buf_len= get_random_string(rand_buffer); + + if (snprintf(buf, HUGE_STRING_LENGTH, "charcol%d = '%.*s'", col_count, + buf_len, rand_buffer) + > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating update\n"); + exit(1); + } + dynstr_append(&update_string, buf); + + if (col_count < num_char_cols) + dynstr_append_mem(&update_string, ",", 1); + } + + if (auto_generate_sql_autoincrement) + { + if (snprintf(buf, HUGE_STRING_LENGTH, " WHERE id = %ld", + (long int)(random() % auto_generate_sql_number) ) + > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating update_string\n"); + exit(1); + } + dynstr_append(&update_string, buf); + } + + if (auto_generate_sql_guid_primary) + { + if (snprintf(buf, HUGE_STRING_LENGTH, " WHERE id = ") > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating update_string\n"); + exit(1); + } + dynstr_append(&update_string, buf); + } + + + ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)); + ptr->string= (char *)my_malloc(update_string.length+1, MYF(MY_WME)); + ptr->length= update_string.length+1; + if (auto_generate_sql_guid_primary) + ptr->type= UPDATE_TYPE_REQUERIES_PREFIX ; + else + ptr->type= UPDATE_TYPE; + strmov(ptr->string, update_string.str); + dynstr_free(&update_string); + DBUG_RETURN(ptr); +} + /* build_insert_string() @@ -661,7 +827,7 @@ build_table_string(void) static statement * build_insert_string(void) { - char buf[RAND_STRING_SIZE]; + char buf[HUGE_STRING_LENGTH]; int col_count; statement *ptr; DYNAMIC_STRING insert_string; @@ -669,30 +835,66 @@ build_insert_string(void) init_dynamic_string(&insert_string, "", 1024, 1024); - dynstr_append_mem(&insert_string, "INSERT INTO t1 VALUES (", 23); - for (col_count= 1; col_count <= num_int_cols; col_count++) + dynstr_append(&insert_string, "INSERT INTO t1 VALUES ("); + + if (auto_generate_sql_autoincrement) { - sprintf(buf, "%ld", random()); + if (snprintf(buf, HUGE_STRING_LENGTH, "NULL") > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating insert\n"); + exit(1); + } dynstr_append(&insert_string, buf); - if (col_count < num_int_cols || num_char_cols > 0) - dynstr_append_mem(&insert_string, ",", 1); + if (num_int_cols || num_char_cols) + dynstr_append(&insert_string, ","); } - for (col_count= 1; col_count <= num_char_cols; col_count++) - { - int buf_len= get_random_string(buf); - dynstr_append_mem(&insert_string, "'", 1); - dynstr_append_mem(&insert_string, buf, buf_len); - dynstr_append_mem(&insert_string, "'", 1); - if (col_count < num_char_cols) - dynstr_append_mem(&insert_string, ",", 1); + if (auto_generate_sql_guid_primary) + { + if (snprintf(buf, HUGE_STRING_LENGTH, "uuid()") > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in create table\n"); + exit(1); + } + dynstr_append(&insert_string, buf); + + if (num_int_cols || num_char_cols) + dynstr_append(&insert_string, ","); } + + if (num_int_cols) + for (col_count= 1; col_count <= num_int_cols; col_count++) + { + if (snprintf(buf, HUGE_STRING_LENGTH, "%ld", random()) > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating insert\n"); + exit(1); + } + dynstr_append(&insert_string, buf); + + if (col_count < num_int_cols || num_char_cols > 0) + dynstr_append_mem(&insert_string, ",", 1); + } + + if (num_char_cols) + for (col_count= 1; col_count <= num_char_cols; col_count++) + { + int buf_len= get_random_string(buf); + dynstr_append_mem(&insert_string, "'", 1); + dynstr_append_mem(&insert_string, buf, buf_len); + dynstr_append_mem(&insert_string, "'", 1); + + if (col_count < num_char_cols) + dynstr_append_mem(&insert_string, ",", 1); + } + dynstr_append_mem(&insert_string, ")", 1); ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)); ptr->string= (char *)my_malloc(insert_string.length+1, MYF(MY_WME)); ptr->length= insert_string.length+1; + ptr->type= INSERT_TYPE; strmov(ptr->string, insert_string.str); dynstr_free(&insert_string); DBUG_RETURN(ptr); @@ -708,7 +910,7 @@ build_insert_string(void) static statement * build_query_string(void) { - char buf[512]; + char buf[HUGE_STRING_LENGTH]; int col_count; statement *ptr; static DYNAMIC_STRING query_string; @@ -719,7 +921,12 @@ build_query_string(void) dynstr_append_mem(&query_string, "SELECT ", 7); for (col_count= 1; col_count <= num_int_cols; col_count++) { - sprintf(buf, "intcol%d", col_count); + if (snprintf(buf, HUGE_STRING_LENGTH, "intcol%d", col_count) + > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating select\n"); + exit(1); + } dynstr_append(&query_string, buf); if (col_count < num_int_cols || num_char_cols > 0) @@ -728,7 +935,12 @@ build_query_string(void) } for (col_count= 1; col_count <= num_char_cols; col_count++) { - sprintf(buf, "charcol%d", col_count); + if (snprintf(buf, HUGE_STRING_LENGTH, "charcol%d", col_count) + > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating select\n"); + exit(1); + } dynstr_append(&query_string, buf); if (col_count < num_char_cols) @@ -739,6 +951,7 @@ build_query_string(void) ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)); ptr->string= (char *)my_malloc(query_string.length+1, MYF(MY_WME)); ptr->length= query_string.length+1; + ptr->type= SELECT_TYPE; strmov(ptr->string, query_string.str); dynstr_free(&query_string); DBUG_RETURN(ptr); @@ -773,6 +986,14 @@ get_options(int *argc,char ***argv) exit(1); } + if (auto_generate_sql && auto_generate_sql_guid_primary && auto_generate_sql_autoincrement) + { + fprintf(stderr, + "%s: Either auto-generate-sql-guid-primary or auto-generate-sql-add-autoincrement can be used!\n", + my_progname); + exit(1); + } + parse_comma(concurrency_str ? concurrency_str : "1", &concurrency); if (lock_directory) @@ -836,6 +1057,23 @@ get_options(int *argc,char ***argv) ptr_statement->next= build_insert_string(); } } + else if (auto_generate_sql_type[0] == 'u') + { + for (ptr_statement= create_statements, x= 0; + x < auto_generate_sql_number; + x++, ptr_statement= ptr_statement->next) + { + ptr_statement->next= build_insert_string(); + } + + query_statements= build_update_string(); + for (ptr_statement= query_statements, x= 0; + x < auto_generate_sql_number; + x++, ptr_statement= ptr_statement->next) + { + ptr_statement->next= build_update_string(); + } + } else /* Mixed mode is default */ { int coin= 0; @@ -943,6 +1181,59 @@ static int run_query(MYSQL *mysql, const char *query, int len) } +static int +generate_primary_key_list(MYSQL *mysql) +{ + char query[HUGE_STRING_LENGTH]; + int len; + MYSQL_RES *result; + MYSQL_ROW row; + char *ptr; + DBUG_ENTER("create_schema"); + + len= snprintf(query, HUGE_STRING_LENGTH, "SELECT id from t1"); + + if (run_query(mysql, query, len)) + { + fprintf(stderr,"%s: Cannot select GUID primary keys. (%s)\n", my_progname, + mysql_error(mysql)); + exit(1); + } + + if (opt_only_print) + { + primary_keys_number_of= 1; + primary_keys= (char *)my_malloc(sizeof(char) * primary_keys_number_of * 32, + MYF(MY_ZEROFILL)); + memcpy(primary_keys, "796c4422-1d94-102a-9d6d-00e0812d", 32); + DBUG_RETURN(0); + } + result= mysql_store_result(mysql); + DBUG_ASSERT(result); + primary_keys_number_of= mysql_num_rows(result); + /* + We know the GUID are 32 characters in length, so we preallocate. + */ + primary_keys= (char *)my_malloc(sizeof(char) * primary_keys_number_of * 32, + MYF(MY_ZEROFILL)); + ptr= primary_keys; + while ((row = mysql_fetch_row(result))) + { + memcpy(ptr, row[0], 32); + ptr+= 32; + } + DBUG_ASSERT(ptr == primary_keys + (primary_keys_number_of * 32)); + mysql_free_result(result); + + DBUG_RETURN(0); +} + +static int +drop_primary_key_list(void) +{ + my_free(primary_keys, MYF(0)); + return 0; +} static int create_schema(MYSQL *mysql, const char *db, statement *stmt, @@ -1219,11 +1510,33 @@ run_task(thread_context *con) limit_not_met: for (ptr= con->stmt; ptr && ptr->length; ptr= ptr->next) { - if (run_query(mysql, ptr->string, ptr->length)) + /* + We have to execute differently based on query type. This should become a function. + */ + if (ptr->type == UPDATE_TYPE_REQUERIES_PREFIX) { - fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", - my_progname, (uint)ptr->length, ptr->string, mysql_error(mysql)); - goto end; + char buffer[HUGE_STRING_LENGTH]; + char *key= primary_keys + (random() % primary_keys_number_of); + int length; + + length= snprintf(buffer, HUGE_STRING_LENGTH, "%.*s '%.*s'", + (int)ptr->length, ptr->string, 32, key); + + if (run_query(mysql, buffer, length)) + { + fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", + my_progname, (uint)length, buffer, mysql_error(mysql)); + goto end; + } + } + else + { + if (run_query(mysql, ptr->string, ptr->length)) + { + fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", + my_progname, (uint)ptr->length, ptr->string, mysql_error(mysql)); + goto end; + } } if (mysql_field_count(mysql)) { diff --git a/mysql-test/t/mysqlslap.test b/mysql-test/t/mysqlslap.test index 01add0d7da8..c98da4b4a0f 100644 --- a/mysql-test/t/mysqlslap.test +++ b/mysql-test/t/mysqlslap.test @@ -14,3 +14,17 @@ --exec $MYSQL_SLAP --silent --concurrency=5 --iterations=20 --delimiter=";" --query="select * from t1;select * from t2" --create="CREATE TABLE t1 (id int, name varchar(64)); create table t2(foo1 varchar(32), foo2 varchar(32)); INSERT INTO t1 VALUES (1, 'This is a test'); insert into t2 values ('test', 'test2')" --exec $MYSQL_SLAP --silent --concurrency=5 --iterations=20 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --create-schema=test_env + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=20 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --create-schema=test_env --auto-generate-sql-add-autoincrement + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-add-autoincrement + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-add-autoincrement --auto-generate-sql-load-type=update + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-add-autoincrement --auto-generate-sql-load-type=read + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-add-autoincrement --auto-generate-sql-load-type=write + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-add-autoincrement --auto-generate-sql-load-type=mixed + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --auto-generate-sql-load-type=update From 465a31bc5933563f9398ea60ce6096cef740d0ca Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 12 Mar 2007 10:25:11 -0700 Subject: [PATCH 02/11] Cleaned up memory allocation so that in all cases of malloc failure the application ends execution. Default values for auto-generate have been made available for adjusting. Key test for rights now works with either type of keys. client/client_priv.h: Adding options for execution defines client/mysqlslap.c: Cleaning up memory allocation so that malloc failures die. Fixed execution so that defaults are not adjustable. mysql-test/t/mysqlslap.test: Adding a test for defaults. --- client/client_priv.h | 3 + client/mysqlslap.c | 420 +++++++++++++++++++++++++----------- mysql-test/t/mysqlslap.test | 4 + 3 files changed, 298 insertions(+), 129 deletions(-) diff --git a/client/client_priv.h b/client/client_priv.h index c8cf68bfd13..00ef1a79d96 100644 --- a/client/client_priv.h +++ b/client/client_priv.h @@ -60,6 +60,9 @@ enum options_client OPT_SLAP_AUTO_GENERATE_SQL_LOAD_TYPE, OPT_SLAP_AUTO_GENERATE_WRITE_NUM, OPT_SLAP_AUTO_GENERATE_ADD_AUTO, OPT_SLAP_AUTO_GENERATE_GUID_PRIMARY, + OPT_SLAP_AUTO_GENERATE_EXECUTE_QUERIES, + OPT_SLAP_AUTO_GENERATE_UNIQUE_WRITE_NUM, + OPT_SLAP_AUTO_GENERATE_UNIQUE_QUERY_NUM, OPT_MYSQL_REPLACE_INTO, OPT_BASE64_OUTPUT, OPT_SERVER_ID, OPT_FIX_TABLE_NAMES, OPT_FIX_DB_NAMES, OPT_SSL_VERIFY_SERVER_CERT, OPT_DEBUG_INFO, OPT_COLUMN_TYPES diff --git a/client/mysqlslap.c b/client/mysqlslap.c index 047193495d2..1ec8d124a48 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -78,8 +78,9 @@ TODO: #define SELECT_TYPE 0 #define UPDATE_TYPE 1 #define INSERT_TYPE 2 -#define UPDATE_TYPE_REQUERIES_PREFIX 3 +#define UPDATE_TYPE_REQUIRES_PREFIX 3 #define CREATE_TABLE_TYPE 4 +#define SELECT_TYPE_REQUIRES_PREFIX 5 #include "client_priv.h" #ifdef HAVE_LIBPTHREAD @@ -114,7 +115,7 @@ static char *shared_memory_base_name=0; static char **defaults_argv; -char *primary_keys; +char **primary_keys; unsigned long long primary_keys_number_of; static char *host= NULL, *opt_password= NULL, *user= NULL, @@ -145,9 +146,12 @@ const char *auto_generate_sql_type= "mixed"; static unsigned long connect_flags= CLIENT_MULTI_RESULTS; static int verbose, num_int_cols, num_char_cols, delimiter_length; -static int iterations; +static unsigned int iterations; static char *default_charset= (char*) MYSQL_DEFAULT_CHARSET_NAME; static ulonglong actual_queries= 0; +static ulonglong auto_actual_queries; +static ulonglong auto_generate_sql_unique_write_number; +static ulonglong auto_generate_sql_unique_query_number; static ulonglong num_of_query; static ulonglong auto_generate_sql_number; const char *concurrency_str= NULL; @@ -162,7 +166,7 @@ static uint opt_protocol= 0; static int get_options(int *argc,char ***argv); static uint opt_mysql_port= 0; -static uint opt_use_threads; +static my_bool opt_use_threads; static const char *load_default_groups[]= { "mysqlslap","client",0 }; @@ -220,8 +224,8 @@ uint get_random_string(char *buf); static statement *build_table_string(void); static statement *build_insert_string(void); static statement *build_update_string(void); -static statement *build_query_string(void); -static int generate_primary_key_list(MYSQL *mysql); +static statement * build_select_string(my_bool key); +static int generate_primary_key_list(MYSQL *mysql, statement *engine_stmt); static int drop_primary_key_list(void); static int create_schema(MYSQL *mysql, const char *db, statement *stmt, statement *engine_stmt); @@ -262,7 +266,7 @@ static int gettimeofday(struct timeval *tp, void *tzp) int main(int argc, char **argv) { MYSQL mysql; - int x; + unsigned int x; unsigned long long client_limit; statement *eptr; @@ -333,16 +337,22 @@ int main(int argc, char **argv) uint *current; conclusions conclusion; + if (verbose >= 2) + printf("Starting Concurrency Test\n"); + for (current= concurrency; current && *current; current++) { stats *head_sptr; stats *sptr; - head_sptr= (stats *)my_malloc(sizeof(stats) * iterations, MYF(MY_ZEROFILL)); + head_sptr= (stats *)my_malloc(sizeof(stats) * iterations, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); bzero(&conclusion, sizeof(conclusions)); - if (num_of_query) + if (auto_actual_queries) + client_limit= auto_actual_queries; + else if (num_of_query) client_limit= num_of_query / *current; else client_limit= actual_queries; @@ -356,6 +366,7 @@ int main(int argc, char **argv) */ if (!opt_preserve) drop_schema(&mysql, create_schema_string); + /* First we create */ if (create_statements) create_schema(&mysql, create_schema_string, create_statements, eptr); @@ -364,12 +375,21 @@ int main(int argc, char **argv) If we generated GUID we need to build a list of them from creation that we can later use. */ - if (auto_generate_sql_guid_primary) - generate_primary_key_list(&mysql); + if (verbose >= 2) + printf("Generating primary key list\n"); + if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) + generate_primary_key_list(&mysql, eptr); run_scheduler(sptr, query_statements, *current, client_limit); + + /* We are finished with this run */ + if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) + drop_primary_key_list(); } + if (verbose >= 2) + printf("Generating stats\n"); + generate_stats(&conclusion, eptr, head_sptr); if (!opt_silent) @@ -377,15 +397,12 @@ int main(int argc, char **argv) if (opt_csv_str) print_conclusions_csv(&conclusion); - my_free((byte *)head_sptr, MYF(0)); + my_free((gptr)head_sptr, MYF(0)); } if (!opt_preserve) drop_schema(&mysql, create_schema_string); - if (auto_generate_sql_guid_primary) - drop_primary_key_list(); - } while (eptr ? (eptr= eptr->next) : 0); if (!opt_only_print) @@ -397,9 +414,9 @@ int main(int argc, char **argv) /* now free all the strings we created */ if (opt_password) - my_free(opt_password, MYF(0)); + my_free((gptr)opt_password, MYF(0)); - my_free((byte *)concurrency, MYF(0)); + my_free((gptr)concurrency, MYF(0)); statement_cleanup(create_statements); statement_cleanup(engine_statements); @@ -407,7 +424,7 @@ int main(int argc, char **argv) #ifdef HAVE_SMEM if (shared_memory_base_name) - my_free(shared_memory_base_name,MYF(MY_ALLOW_ZERO_PTR)); + my_free((gptr)shared_memory_base_name, MYF(MY_ALLOW_ZERO_PTR)); #endif free_defaults(defaults_argv); my_end(0); @@ -429,6 +446,10 @@ static struct my_option my_long_options[] = (gptr*) &auto_generate_sql_autoincrement, (gptr*) &auto_generate_sql_autoincrement, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"auto-generate-sql-execute-number", OPT_SLAP_AUTO_GENERATE_EXECUTE_QUERIES, + "Set this number to generate a set number of queries to run.\n", + (gptr*) &auto_actual_queries, (gptr*) &auto_actual_queries, + 0, GET_ULL, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"auto-generate-sql-guid-primary", OPT_SLAP_AUTO_GENERATE_GUID_PRIMARY, "Add GUID based primary keys to auto-generated tables.", (gptr*) &auto_generate_sql_guid_primary, @@ -438,6 +459,18 @@ static struct my_option my_long_options[] = "Load types are mixed, update, write, or read. Default is mixed\n", (gptr*) &auto_generate_sql_type, (gptr*) &auto_generate_sql_type, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"auto-generate-sql-unique-query-number", + OPT_SLAP_AUTO_GENERATE_UNIQUE_QUERY_NUM, + "Number of unique queries auto tests", + (gptr*) &auto_generate_sql_unique_query_number, + (gptr*) &auto_generate_sql_unique_query_number, + 0, GET_ULL, REQUIRED_ARG, 10, 0, 0, 0, 0, 0}, + {"auto-generate-sql-unique-write-number", + OPT_SLAP_AUTO_GENERATE_UNIQUE_WRITE_NUM, + "Number of unique queries for auto-generate-sql-write-number", + (gptr*) &auto_generate_sql_unique_write_number, + (gptr*) &auto_generate_sql_unique_write_number, + 0, GET_ULL, REQUIRED_ARG, 10, 0, 0, 0, 0, 0}, {"auto-generate-sql-write-number", OPT_SLAP_AUTO_GENERATE_WRITE_NUM, "Number of rows to insert to used in read and write loads (default is 100).\n", (gptr*) &auto_generate_sql_number, (gptr*) &auto_generate_sql_number, @@ -532,7 +565,7 @@ static struct my_option my_long_options[] = {"use-threads", OPT_USE_THREADS, "Use pthread calls instead of fork() calls (default on Windows)", (gptr*) &opt_use_threads, (gptr*) &opt_use_threads, 0, - GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, #ifndef DONT_ALLOW_USER_CHANGE {"user", 'u', "User for login if not current user.", (gptr*) &user, (gptr*) &user, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -589,7 +622,7 @@ get_one_option(int optid, const struct my_option *opt __attribute__((unused)), if (argument) { char *start= argument; - my_free(opt_password,MYF(MY_ALLOW_ZERO_PTR)); + my_free((gptr)opt_password, MYF(MY_ALLOW_ZERO_PTR)); opt_password= my_strdup(argument,MYF(MY_FAE)); while (*argument) *argument++= 'x'; /* Destroy argument */ if (*start) @@ -721,8 +754,10 @@ build_table_string(void) } dynstr_append(&table_string, ")"); - ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)); - ptr->string = (char *)my_malloc(table_string.length+1, MYF(MY_WME)); + ptr= (statement *)my_malloc(sizeof(statement), + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); + ptr->string = (char *)my_malloc(table_string.length+1, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); ptr->length= table_string.length+1; strmov(ptr->string, table_string.str); dynstr_free(&table_string); @@ -782,19 +817,7 @@ build_update_string(void) dynstr_append_mem(&update_string, ",", 1); } - if (auto_generate_sql_autoincrement) - { - if (snprintf(buf, HUGE_STRING_LENGTH, " WHERE id = %ld", - (long int)(random() % auto_generate_sql_number) ) - > HUGE_STRING_LENGTH) - { - fprintf(stderr, "Memory Allocation error in creating update_string\n"); - exit(1); - } - dynstr_append(&update_string, buf); - } - - if (auto_generate_sql_guid_primary) + if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) { if (snprintf(buf, HUGE_STRING_LENGTH, " WHERE id = ") > HUGE_STRING_LENGTH) { @@ -805,11 +828,14 @@ build_update_string(void) } - ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)); - ptr->string= (char *)my_malloc(update_string.length+1, MYF(MY_WME)); + ptr= (statement *)my_malloc(sizeof(statement), + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); + + ptr->string= (char *)my_malloc(update_string.length + 1, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); ptr->length= update_string.length+1; - if (auto_generate_sql_guid_primary) - ptr->type= UPDATE_TYPE_REQUERIES_PREFIX ; + if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) + ptr->type= UPDATE_TYPE_REQUIRES_PREFIX ; else ptr->type= UPDATE_TYPE; strmov(ptr->string, update_string.str); @@ -891,8 +917,10 @@ build_insert_string(void) dynstr_append_mem(&insert_string, ")", 1); - ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)); - ptr->string= (char *)my_malloc(insert_string.length+1, MYF(MY_WME)); + ptr= (statement *)my_malloc(sizeof(statement), + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); + ptr->string= (char *)my_malloc(insert_string.length + 1, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); ptr->length= insert_string.length+1; ptr->type= INSERT_TYPE; strmov(ptr->string, insert_string.str); @@ -902,19 +930,19 @@ build_insert_string(void) /* - build_query_string() + build_select_string() This function builds a query if the user opts to not supply a query statement or file containing a query statement */ static statement * -build_query_string(void) +build_select_string(my_bool key) { char buf[HUGE_STRING_LENGTH]; int col_count; statement *ptr; static DYNAMIC_STRING query_string; - DBUG_ENTER("build_query_string"); + DBUG_ENTER("build_select_string"); init_dynamic_string(&query_string, "", 1024, 1024); @@ -947,11 +975,22 @@ build_query_string(void) dynstr_append_mem(&query_string, ",", 1); } - dynstr_append_mem(&query_string, " FROM t1", 8); - ptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)); - ptr->string= (char *)my_malloc(query_string.length+1, MYF(MY_WME)); + dynstr_append(&query_string, " FROM t1"); + + if ((key) && + (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary)) + dynstr_append(&query_string, " WHERE id = "); + + ptr= (statement *)my_malloc(sizeof(statement), + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); + ptr->string= (char *)my_malloc(query_string.length + 1, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); ptr->length= query_string.length+1; - ptr->type= SELECT_TYPE; + if ((key) && + (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary)) + ptr->type= SELECT_TYPE_REQUIRES_PREFIX; + else + ptr->type= SELECT_TYPE; strmov(ptr->string, query_string.str); dynstr_free(&query_string); DBUG_RETURN(ptr); @@ -986,7 +1025,8 @@ get_options(int *argc,char ***argv) exit(1); } - if (auto_generate_sql && auto_generate_sql_guid_primary && auto_generate_sql_autoincrement) + if (auto_generate_sql && auto_generate_sql_guid_primary && + auto_generate_sql_autoincrement) { fprintf(stderr, "%s: Either auto-generate-sql-guid-primary or auto-generate-sql-add-autoincrement can be used!\n", @@ -994,6 +1034,27 @@ get_options(int *argc,char ***argv) exit(1); } + if (auto_generate_sql && + ((auto_generate_sql_autoincrement == FALSE) || + (auto_generate_sql_guid_primary == FALSE)) && + auto_generate_sql_type == 'k') + { + fprintf(stderr, + "%s: Can't perform key test without a primary key!\n", + my_progname); + exit(1); + } + + + + if (auto_generate_sql && num_of_query && auto_actual_queries) + { + fprintf(stderr, + "%s: Either auto-generate-sql-execute-number or number-of-queries can be used!\n", + my_progname); + exit(1); + } + parse_comma(concurrency_str ? concurrency_str : "1", &concurrency); if (lock_directory) @@ -1029,18 +1090,48 @@ get_options(int *argc,char ***argv) unsigned long long x= 0; statement *ptr_statement; + if (verbose >= 2) + printf("Building Create Statements for Auto\n"); + create_statements= build_table_string(); + /* + Pre-populate table + */ + for (ptr_statement= create_statements, x= 0; + x < auto_generate_sql_unique_write_number; + x++, ptr_statement= ptr_statement->next) + { + ptr_statement->next= build_insert_string(); + } + + if (verbose >= 2) + printf("Building Query Statements for Auto\n"); if (auto_generate_sql_type[0] == 'r') { - for (ptr_statement= create_statements, x= 0; - x < auto_generate_sql_number; + if (verbose >= 2) + printf("Generating SELECT Statements for Auto\n"); + + query_statements= build_select_string(FALSE); + for (ptr_statement= query_statements, x= 0; + x < auto_generate_sql_unique_query_number; x++, ptr_statement= ptr_statement->next) { - ptr_statement->next= build_insert_string(); + ptr_statement->next= build_select_string(FALSE); } + } + else if (auto_generate_sql_type[0] == 'k') + { + if (verbose >= 2) + printf("Generating SELECT for keys Statements for Auto\n"); - query_statements= build_query_string(); + query_statements= build_select_string(TRUE); + for (ptr_statement= query_statements, x= 0; + x < auto_generate_sql_unique_query_number; + x++, ptr_statement= ptr_statement->next) + { + ptr_statement->next= build_select_string(TRUE); + } } else if (auto_generate_sql_type[0] == 'w') { @@ -1049,9 +1140,11 @@ get_options(int *argc,char ***argv) Archive (since strings which were identical one after another would be too easily optimized). */ + if (verbose >= 2) + printf("Generating INSERT Statements for Auto\n"); query_statements= build_insert_string(); for (ptr_statement= query_statements, x= 0; - x < auto_generate_sql_number; + x < auto_generate_sql_unique_query_number; x++, ptr_statement= ptr_statement->next) { ptr_statement->next= build_insert_string(); @@ -1059,16 +1152,9 @@ get_options(int *argc,char ***argv) } else if (auto_generate_sql_type[0] == 'u') { - for (ptr_statement= create_statements, x= 0; - x < auto_generate_sql_number; - x++, ptr_statement= ptr_statement->next) - { - ptr_statement->next= build_insert_string(); - } - query_statements= build_update_string(); for (ptr_statement= query_statements, x= 0; - x < auto_generate_sql_number; + x < auto_generate_sql_unique_query_number; x++, ptr_statement= ptr_statement->next) { ptr_statement->next= build_update_string(); @@ -1084,7 +1170,7 @@ get_options(int *argc,char ***argv) at the moment it results in "every other". */ for (ptr_statement= query_statements, x= 0; - x < 4; + x < auto_generate_sql_unique_query_number; x++, ptr_statement= ptr_statement->next) { if (coin) @@ -1094,7 +1180,7 @@ get_options(int *argc,char ***argv) } else { - ptr_statement->next= build_query_string(); + ptr_statement->next= build_select_string(TRUE); coin= 1; } } @@ -1116,12 +1202,13 @@ get_options(int *argc,char ***argv) fprintf(stderr,"%s: Could not open create file\n", my_progname); exit(1); } - tmp_string= (char *)my_malloc(sbuf.st_size+1, MYF(MY_WME)); + tmp_string= (char *)my_malloc(sbuf.st_size + 1, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); my_read(data_file, tmp_string, sbuf.st_size, MYF(0)); tmp_string[sbuf.st_size]= '\0'; my_close(data_file,MYF(0)); parse_delimiter(tmp_string, &create_statements, delimiter[0]); - my_free(tmp_string, MYF(0)); + my_free((gptr)tmp_string, MYF(0)); } else if (create_string) { @@ -1142,14 +1229,15 @@ get_options(int *argc,char ***argv) fprintf(stderr,"%s: Could not open query supplied file\n", my_progname); exit(1); } - tmp_string= (char *)my_malloc(sbuf.st_size+1, MYF(MY_WME)); + tmp_string= (char *)my_malloc(sbuf.st_size + 1, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); my_read(data_file, tmp_string, sbuf.st_size, MYF(0)); tmp_string[sbuf.st_size]= '\0'; my_close(data_file,MYF(0)); if (user_supplied_query) actual_queries= parse_delimiter(tmp_string, &query_statements, delimiter[0]); - my_free(tmp_string, MYF(0)); + my_free((gptr)tmp_string, MYF(0)); } else if (user_supplied_query) { @@ -1158,6 +1246,9 @@ get_options(int *argc,char ***argv) } } + if (verbose >= 2) + printf("Parsing engines to use.\n"); + if (default_engine) parse_delimiter(default_engine, &engine_statements, ','); @@ -1175,55 +1266,65 @@ static int run_query(MYSQL *mysql, const char *query, int len) return 0; } - if (verbose >= 2) + if (verbose >= 3) printf("%.*s;\n", len, query); return mysql_real_query(mysql, query, len); } static int -generate_primary_key_list(MYSQL *mysql) +generate_primary_key_list(MYSQL *mysql, statement *engine_stmt) { char query[HUGE_STRING_LENGTH]; int len; MYSQL_RES *result; MYSQL_ROW row; - char *ptr; + unsigned long long counter; DBUG_ENTER("create_schema"); - len= snprintf(query, HUGE_STRING_LENGTH, "SELECT id from t1"); - - if (run_query(mysql, query, len)) - { - fprintf(stderr,"%s: Cannot select GUID primary keys. (%s)\n", my_progname, - mysql_error(mysql)); - exit(1); - } - - if (opt_only_print) + /* + Blackhole is a special case, this allows us to test the upper end + of the server during load runs. + */ + if (opt_only_print || (engine_stmt && + strstr(engine_stmt->string, "blackhole"))) { primary_keys_number_of= 1; - primary_keys= (char *)my_malloc(sizeof(char) * primary_keys_number_of * 32, - MYF(MY_ZEROFILL)); - memcpy(primary_keys, "796c4422-1d94-102a-9d6d-00e0812d", 32); - DBUG_RETURN(0); + primary_keys= (char **)my_malloc(sizeof(char *) * primary_keys_number_of, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); + /* Yes, we strdup a const string to simplify the interface */ + primary_keys[0]= my_strdup("796c4422-1d94-102a-9d6d-00e0812d", MYF(0)); } - result= mysql_store_result(mysql); - DBUG_ASSERT(result); - primary_keys_number_of= mysql_num_rows(result); - /* - We know the GUID are 32 characters in length, so we preallocate. - */ - primary_keys= (char *)my_malloc(sizeof(char) * primary_keys_number_of * 32, - MYF(MY_ZEROFILL)); - ptr= primary_keys; - while ((row = mysql_fetch_row(result))) + else { - memcpy(ptr, row[0], 32); - ptr+= 32; + len= snprintf(query, HUGE_STRING_LENGTH, "SELECT id from t1"); + + if (run_query(mysql, query, len)) + { + fprintf(stderr,"%s: Cannot select GUID primary keys. (%s)\n", my_progname, + mysql_error(mysql)); + exit(1); + } + + result= mysql_store_result(mysql); + primary_keys_number_of= mysql_num_rows(result); + + /* So why check this? Blackhole :) */ + if (primary_keys_number_of) + { + /* + We create the structure and loop and create the items. + */ + primary_keys= (char **)my_malloc(sizeof(char *) * primary_keys_number_of, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); + row= mysql_fetch_row(result); + for (counter= 0; counter < primary_keys_number_of; + counter++, row= mysql_fetch_row(result)) + primary_keys[counter]= my_strdup(row[0], MYF(0)); + } + + mysql_free_result(result); } - DBUG_ASSERT(ptr == primary_keys + (primary_keys_number_of * 32)); - mysql_free_result(result); DBUG_RETURN(0); } @@ -1231,7 +1332,16 @@ generate_primary_key_list(MYSQL *mysql) static int drop_primary_key_list(void) { - my_free(primary_keys, MYF(0)); + unsigned long long counter; + + if (primary_keys_number_of) + { + for (counter= 0; counter < primary_keys_number_of; counter++) + my_free((gptr)primary_keys[counter], MYF(0)); + + my_free((gptr)primary_keys, MYF(0)); + } + return 0; } @@ -1241,11 +1351,16 @@ create_schema(MYSQL *mysql, const char *db, statement *stmt, { char query[HUGE_STRING_LENGTH]; statement *ptr; + statement *after_create; int len; + ulonglong count; DBUG_ENTER("create_schema"); len= snprintf(query, HUGE_STRING_LENGTH, "CREATE SCHEMA `%s`", db); + if (verbose >= 2) + printf("Loading Pre-data\n"); + if (run_query(mysql, query, len)) { fprintf(stderr,"%s: Cannot create schema %s : %s\n", my_progname, db, @@ -1259,8 +1374,9 @@ create_schema(MYSQL *mysql, const char *db, statement *stmt, } else { - if (verbose >= 2) + if (verbose >= 3) printf("%s;\n", query); + if (mysql_select_db(mysql, db)) { fprintf(stderr,"%s: Cannot select schema '%s': %s\n",my_progname, db, @@ -1281,8 +1397,15 @@ create_schema(MYSQL *mysql, const char *db, statement *stmt, } } - for (ptr= stmt; ptr && ptr->length; ptr= ptr->next) + count= 0; + after_create= stmt; + +limit_not_met: + for (ptr= after_create; ptr && ptr->length; ptr= ptr->next, count++) { + if (auto_generate_sql && ( auto_generate_sql_number == count)) + break; + if (run_query(mysql, ptr->string, ptr->length)) { fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", @@ -1291,6 +1414,13 @@ create_schema(MYSQL *mysql, const char *db, statement *stmt, } } + if (auto_generate_sql && (auto_generate_sql_number > count )) + { + /* Special case for auto create, we don't want to create tables twice */ + after_create= stmt->next; + goto limit_not_met; + } + DBUG_RETURN(0); } @@ -1379,10 +1509,9 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) /* child */ DBUG_PRINT("info", ("fork returned 0, calling task(\"%s\"), pid %d gid %d", stmts ? stmts->string : "", pid, getgid())); - if (verbose >= 2) - fprintf(stderr, - "%s: fork returned 0, calling task pid %d gid %d\n", - my_progname, pid, getgid()); + if (verbose >= 3) + printf("%s: fork returned 0, calling task pid %d gid %d\n", + my_progname, pid, getgid()); run_task(&con); exit(0); break; @@ -1398,9 +1527,9 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) default: /* parent, forked */ DBUG_PRINT("info", ("default, break: pid %d gid %d", pid, getgid())); - if (verbose >= 2) - fprintf(stderr,"%s: fork returned %d, gid %d\n", - my_progname, pid, getgid()); + if (verbose >= 3) + printf("%s: fork returned %d, gid %d\n", + my_progname, pid, getgid()); break; } } @@ -1437,8 +1566,11 @@ WAIT: pid= wait(&status); DBUG_PRINT("info", ("Parent: child %d status %d", pid, status)); if (status != 0) + { printf("%s: Child %d died with the status %d\n", my_progname, pid, status); + exit(0); + } } } #endif @@ -1468,10 +1600,18 @@ run_task(thread_context *con) DBUG_PRINT("info", ("task script \"%s\"", con->stmt ? con->stmt->string : "")); if (!(mysql= mysql_init(NULL))) - goto end; + { + fprintf(stderr,"%s: mysql_init() failed ERROR : %s\n", + my_progname, mysql_error(mysql)); + exit(0); + } if (con->thread && mysql_thread_init()) - goto end; + { + fprintf(stderr,"%s: mysql_thread_init() failed ERROR : %s\n", + my_progname, mysql_error(mysql)); + exit(0); + } DBUG_PRINT("info", ("trying to connect to host %s as user %s", host, user)); lock_file= my_open(lock_file_str, O_RDWR, MYF(0)); @@ -1504,7 +1644,7 @@ run_task(thread_context *con) } DBUG_PRINT("info", ("connected.")); if (verbose >= 3) - fprintf(stderr, "connected!\n"); + printf("connected!\n"); queries= 0; limit_not_met: @@ -1513,20 +1653,38 @@ limit_not_met: /* We have to execute differently based on query type. This should become a function. */ - if (ptr->type == UPDATE_TYPE_REQUERIES_PREFIX) + if ((ptr->type == UPDATE_TYPE_REQUIRES_PREFIX) || + (ptr->type == SELECT_TYPE_REQUIRES_PREFIX)) { - char buffer[HUGE_STRING_LENGTH]; - char *key= primary_keys + (random() % primary_keys_number_of); int length; + long int key_val; + char *key; + char buffer[HUGE_STRING_LENGTH]; - length= snprintf(buffer, HUGE_STRING_LENGTH, "%.*s '%.*s'", - (int)ptr->length, ptr->string, 32, key); - - if (run_query(mysql, buffer, length)) + /* + This should only happen if some sort of new engine was + implemented that didn't properly handle UPDATEs. + + Just in case someone runs this under an experimental engine we don't + want a crash so the if() is placed here. + */ + DBUG_ASSERT(primary_keys_number_of); + if (primary_keys_number_of) { - fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", - my_progname, (uint)length, buffer, mysql_error(mysql)); - goto end; + key_val= random() % primary_keys_number_of; + key= primary_keys[key_val]; + + DBUG_ASSERT(key); + + length= snprintf(buffer, HUGE_STRING_LENGTH, "%.*s '%s'", + (int)ptr->length, ptr->string, key); + + if (run_query(mysql, buffer, length)) + { + fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", + my_progname, (uint)length, buffer, mysql_error(mysql)); + exit(0); + } } } else @@ -1535,9 +1693,10 @@ limit_not_met: { fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", my_progname, (uint)ptr->length, ptr->string, mysql_error(mysql)); - goto end; + exit(0); } } + if (mysql_field_count(mysql)) { result= mysql_store_result(mysql); @@ -1581,9 +1740,11 @@ parse_delimiter(const char *script, statement **stmt, char delm) uint length= strlen(script); uint count= 0; /* We know that there is always one */ - for (tmp= *sptr= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)); + for (tmp= *sptr= (statement *)my_malloc(sizeof(statement), + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); (retstr= strchr(ptr, delm)); - tmp->next= (statement *)my_malloc(sizeof(statement), MYF(MY_ZEROFILL)), + tmp->next= (statement *)my_malloc(sizeof(statement), + MYF(MY_ZEROFILL|MY_FAE|MY_WME)), tmp= tmp->next) { count++; @@ -1619,7 +1780,8 @@ parse_comma(const char *string, uint **range) if (*ptr == ',') count++; /* One extra spot for the NULL */ - nptr= *range= (uint *)my_malloc(sizeof(uint) * (count + 1), MYF(MY_ZEROFILL)); + nptr= *range= (uint *)my_malloc(sizeof(uint) * (count + 1), + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); ptr= (char *)string; x= 0; @@ -1670,7 +1832,7 @@ void generate_stats(conclusions *con, statement *eng, stats *sptr) { stats *ptr; - int x; + unsigned int x; con->min_timing= sptr->timing; con->max_timing= sptr->timing; @@ -1710,7 +1872,7 @@ statement_cleanup(statement *stmt) { nptr= ptr->next; if (ptr->string) - my_free(ptr->string, MYF(0)); - my_free((byte *)ptr, MYF(0)); + my_free((gptr)ptr->string, MYF(0)); + my_free((gptr)(byte *)ptr, MYF(0)); } } diff --git a/mysql-test/t/mysqlslap.test b/mysql-test/t/mysqlslap.test index c98da4b4a0f..5d98dac03ea 100644 --- a/mysql-test/t/mysqlslap.test +++ b/mysql-test/t/mysqlslap.test @@ -28,3 +28,7 @@ --exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-add-autoincrement --auto-generate-sql-load-type=mixed --exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --auto-generate-sql-load-type=update + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --auto-generate-sql-load-type=update --auto-generate-sql-execute-number=5 + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --auto-generate-sql-load-type=key --auto-generate-sql-execute-number=5 From c573f816b7a5ae1b21310e490a4fd6ceacdf6b58 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 13 Mar 2007 00:56:30 -0700 Subject: [PATCH 03/11] Cleaning up 4 pushbuild warnings caught by Windows. client/mysqlslap.c: Cleanup of pushbuild warnings. --- client/mysqlslap.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/client/mysqlslap.c b/client/mysqlslap.c index 1ec8d124a48..1976c54c393 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -1037,7 +1037,7 @@ get_options(int *argc,char ***argv) if (auto_generate_sql && ((auto_generate_sql_autoincrement == FALSE) || (auto_generate_sql_guid_primary == FALSE)) && - auto_generate_sql_type == 'k') + auto_generate_sql_type[0] == 'k') { fprintf(stderr, "%s: Can't perform key test without a primary key!\n", @@ -1290,7 +1290,8 @@ generate_primary_key_list(MYSQL *mysql, statement *engine_stmt) strstr(engine_stmt->string, "blackhole"))) { primary_keys_number_of= 1; - primary_keys= (char **)my_malloc(sizeof(char *) * primary_keys_number_of, + primary_keys= (char **)my_malloc((uint)(sizeof(char *) * + primary_keys_number_of), MYF(MY_ZEROFILL|MY_FAE|MY_WME)); /* Yes, we strdup a const string to simplify the interface */ primary_keys[0]= my_strdup("796c4422-1d94-102a-9d6d-00e0812d", MYF(0)); @@ -1315,7 +1316,8 @@ generate_primary_key_list(MYSQL *mysql, statement *engine_stmt) /* We create the structure and loop and create the items. */ - primary_keys= (char **)my_malloc(sizeof(char *) * primary_keys_number_of, + primary_keys= (char **)my_malloc((uint)(sizeof(char *) * + primary_keys_number_of), MYF(MY_ZEROFILL|MY_FAE|MY_WME)); row= mysql_fetch_row(result); for (counter= 0; counter < primary_keys_number_of; @@ -1657,7 +1659,7 @@ limit_not_met: (ptr->type == SELECT_TYPE_REQUIRES_PREFIX)) { int length; - long int key_val; + unsigned int key_val; char *key; char buffer[HUGE_STRING_LENGTH]; @@ -1671,7 +1673,7 @@ limit_not_met: DBUG_ASSERT(primary_keys_number_of); if (primary_keys_number_of) { - key_val= random() % primary_keys_number_of; + key_val= (unsigned int)(random() % primary_keys_number_of); key= primary_keys[key_val]; DBUG_ASSERT(key); From b1af9693e0f5561142ae6ab1c55ac93de9abf6e5 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 13 Mar 2007 10:27:59 -0700 Subject: [PATCH 04/11] Added comment, and fixed test. client/mysqlslap.c: Comment added (and extended if() so that I didn't have to keep looking at it to make sure it was right). --- client/mysqlslap.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/client/mysqlslap.c b/client/mysqlslap.c index 1976c54c393..1dc49aa06f3 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -1034,16 +1034,19 @@ get_options(int *argc,char ***argv) exit(1); } - if (auto_generate_sql && - ((auto_generate_sql_autoincrement == FALSE) || - (auto_generate_sql_guid_primary == FALSE)) && - auto_generate_sql_type[0] == 'k') - { + /* + We are testing to make sure that if someone specified a key search + that we actually added a key! + */ + if (auto_generate_sql && auto_generate_sql_type[0] == 'k') + if ( auto_generate_sql_autoincrement == FALSE && + auto_generate_sql_guid_primary == FALSE) + { fprintf(stderr, "%s: Can't perform key test without a primary key!\n", my_progname); exit(1); - } + } From 260e43e75dee8a261ab5dd0ab8a595f9acc9f814 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 15 Mar 2007 23:39:07 -0700 Subject: [PATCH 05/11] Correctly report load type. Updated engine to also handle create options Secondary indexes can now be generated (aka the test PeterZ thoughts!) client/client_priv.h: Option for generating secondary indexes client/mysqlslap.c: Option for generating secondary index operations. Cleaned up memory allocation that was wasteful. Engine options are now possible Correctly report test type. mysql-test/t/mysqlslap.test: Additional test for secondary indexes sql/authors.h: Updated for Patrick --- client/client_priv.h | 1 + client/mysqlslap.c | 331 +++++++++++++++++++++++++++++------- mysql-test/t/mysqlslap.test | 2 + sql/authors.h | 1 + 4 files changed, 269 insertions(+), 66 deletions(-) diff --git a/client/client_priv.h b/client/client_priv.h index 00ef1a79d96..b6543870528 100644 --- a/client/client_priv.h +++ b/client/client_priv.h @@ -61,6 +61,7 @@ enum options_client OPT_SLAP_AUTO_GENERATE_ADD_AUTO, OPT_SLAP_AUTO_GENERATE_GUID_PRIMARY, OPT_SLAP_AUTO_GENERATE_EXECUTE_QUERIES, + OPT_SLAP_AUTO_GENERATE_SECONDARY_INDEXES, OPT_SLAP_AUTO_GENERATE_UNIQUE_WRITE_NUM, OPT_SLAP_AUTO_GENERATE_UNIQUE_QUERY_NUM, OPT_MYSQL_REPLACE_INTO, OPT_BASE64_OUTPUT, OPT_SERVER_ID, diff --git a/client/mysqlslap.c b/client/mysqlslap.c index 1dc49aa06f3..0010d8e46ae 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -145,13 +145,22 @@ const char *auto_generate_sql_type= "mixed"; static unsigned long connect_flags= CLIENT_MULTI_RESULTS; -static int verbose, num_int_cols, num_char_cols, delimiter_length; +static int verbose, delimiter_length; +const char *num_int_cols_opt; +const char *num_char_cols_opt; +/* Yes, we do set defaults here */ +static unsigned int num_int_cols= 1; +static unsigned int num_char_cols= 1; +static unsigned int num_int_cols_index= 0; +static unsigned int num_char_cols_index= 0; + static unsigned int iterations; static char *default_charset= (char*) MYSQL_DEFAULT_CHARSET_NAME; static ulonglong actual_queries= 0; static ulonglong auto_actual_queries; static ulonglong auto_generate_sql_unique_write_number; static ulonglong auto_generate_sql_unique_query_number; +static unsigned int auto_generate_sql_secondary_indexes; static ulonglong num_of_query; static ulonglong auto_generate_sql_number; const char *concurrency_str= NULL; @@ -176,9 +185,21 @@ struct statement { char *string; size_t length; unsigned char type; + char *option; + size_t option_length; statement *next; }; +typedef struct option_string option_string; + +struct option_string { + char *string; + size_t length; + char *option; + size_t option_length; + option_string *next; +}; + typedef struct stats stats; struct stats { @@ -209,30 +230,32 @@ struct conclusions { unsigned long long min_rows; }; +static option_string *engine_options= NULL; static statement *create_statements= NULL, - *engine_statements= NULL, *query_statements= NULL; /* Prototypes */ void print_conclusions(conclusions *con); void print_conclusions_csv(conclusions *con); -void generate_stats(conclusions *con, statement *eng, stats *sptr); +void generate_stats(conclusions *con, option_string *eng, stats *sptr); uint parse_comma(const char *string, uint **range); uint parse_delimiter(const char *script, statement **stmt, char delm); +uint parse_option(const char *origin, option_string **stmt, char delm); static int drop_schema(MYSQL *mysql, const char *db); uint get_random_string(char *buf); static statement *build_table_string(void); static statement *build_insert_string(void); static statement *build_update_string(void); static statement * build_select_string(my_bool key); -static int generate_primary_key_list(MYSQL *mysql, statement *engine_stmt); +static int generate_primary_key_list(MYSQL *mysql, option_string *engine_stmt); static int drop_primary_key_list(void); static int create_schema(MYSQL *mysql, const char *db, statement *stmt, - statement *engine_stmt); + option_string *engine_stmt); static int run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit); int run_task(thread_context *con); void statement_cleanup(statement *stmt); +void option_cleanup(option_string *stmt); static const char ALPHANUMERICS[]= "0123456789ABCDEFGHIJKLMNOPQRSTWXYZabcdefghijklmnopqrstuvwxyz"; @@ -268,7 +291,7 @@ int main(int argc, char **argv) MYSQL mysql; unsigned int x; unsigned long long client_limit; - statement *eptr; + option_string *eptr; #ifdef __WIN__ opt_use_threads= 1; @@ -330,7 +353,7 @@ int main(int argc, char **argv) } /* Main iterations loop */ - eptr= engine_statements; + eptr= engine_options; do { /* For the final stage we run whatever queries we were asked to run */ @@ -419,8 +442,8 @@ int main(int argc, char **argv) my_free((gptr)concurrency, MYF(0)); statement_cleanup(create_statements); - statement_cleanup(engine_statements); statement_cleanup(query_statements); + option_cleanup(engine_options); #ifdef HAVE_SMEM if (shared_memory_base_name) @@ -456,9 +479,15 @@ static struct my_option my_long_options[] = (gptr*) &auto_generate_sql_guid_primary, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"auto-generate-sql-load-type", OPT_SLAP_AUTO_GENERATE_SQL_LOAD_TYPE, - "Load types are mixed, update, write, or read. Default is mixed\n", + "Load types are mixed, update, write, key, or read. Default is mixed\n", (gptr*) &auto_generate_sql_type, (gptr*) &auto_generate_sql_type, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"auto-generate-sql-secondary-indexes", + OPT_SLAP_AUTO_GENERATE_SECONDARY_INDEXES, + "Number of secondary indexes to add auto-generated tables.", + (gptr*) &auto_generate_sql_secondary_indexes, + (gptr*) &auto_generate_sql_secondary_indexes, 0, + GET_UINT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"auto-generate-sql-unique-query-number", OPT_SLAP_AUTO_GENERATE_UNIQUE_QUERY_NUM, "Number of unique queries auto tests", @@ -510,12 +539,12 @@ static struct my_option my_long_options[] = REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"number-char-cols", 'x', "Number of VARCHAR columns to create table with if specifying --auto-generate-sql ", - (gptr*) &num_char_cols, (gptr*) &num_char_cols, 0, GET_UINT, REQUIRED_ARG, - 1, 0, 0, 0, 0, 0}, + (gptr*) &num_char_cols_opt, (gptr*) &num_char_cols_opt, 0, GET_STR, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, {"number-int-cols", 'y', "Number of INT columns to create table with if specifying --auto-generate-sql.", - (gptr*) &num_int_cols, (gptr*) &num_int_cols, 0, GET_UINT, REQUIRED_ARG, - 1, 0, 0, 0, 0, 0}, + (gptr*) &num_int_cols_opt, (gptr*) &num_int_cols_opt, 0, GET_STR, REQUIRED_ARG, + 0, 0, 0, 0, 0, 0}, {"number-of-queries", OPT_MYSQL_NUMBER_OF_QUERY, "Limit each client to this number of queries (this is not exact).", (gptr*) &num_of_query, (gptr*) &num_of_query, 0, @@ -685,12 +714,12 @@ static statement * build_table_string(void) { char buf[HUGE_STRING_LENGTH]; - int col_count; + unsigned int col_count; statement *ptr; DYNAMIC_STRING table_string; DBUG_ENTER("build_table_string"); - DBUG_PRINT("info", ("num int cols %d num char cols %d", + DBUG_PRINT("info", ("num int cols %u num char cols %u", num_int_cols, num_char_cols)); init_dynamic_string(&table_string, "", 1024, 1024); @@ -699,12 +728,7 @@ build_table_string(void) if (auto_generate_sql_autoincrement) { - if (snprintf(buf, HUGE_STRING_LENGTH, "id serial") > HUGE_STRING_LENGTH) - { - fprintf(stderr, "Memory Allocation error in create table\n"); - exit(1); - } - dynstr_append(&table_string, buf); + dynstr_append(&table_string, "id serial"); if (num_int_cols || num_char_cols) dynstr_append(&table_string, ","); @@ -712,12 +736,29 @@ build_table_string(void) if (auto_generate_sql_guid_primary) { - if (snprintf(buf, HUGE_STRING_LENGTH, "id varchar(32) primary key") > HUGE_STRING_LENGTH) + dynstr_append(&table_string, "id varchar(32) primary key"); + + if (num_int_cols || num_char_cols || auto_generate_sql_guid_primary) + dynstr_append(&table_string, ","); + } + + if (auto_generate_sql_secondary_indexes) + { + unsigned int count; + + for (count= 0; count < auto_generate_sql_secondary_indexes; count++) { + if (count) /* Except for the first pass we add a comma */ + dynstr_append(&table_string, ","); + + if (snprintf(buf, HUGE_STRING_LENGTH, "id%d varchar(32) unique key", count) + > HUGE_STRING_LENGTH) + { fprintf(stderr, "Memory Allocation error in create table\n"); exit(1); + } + dynstr_append(&table_string, buf); } - dynstr_append(&table_string, buf); if (num_int_cols || num_char_cols) dynstr_append(&table_string, ","); @@ -726,11 +767,23 @@ build_table_string(void) if (num_int_cols) for (col_count= 1; col_count <= num_int_cols; col_count++) { - if (snprintf(buf, HUGE_STRING_LENGTH, "intcol%d INT(32)", col_count) - > HUGE_STRING_LENGTH) + if (num_int_cols_index) { - fprintf(stderr, "Memory Allocation error in create table\n"); - exit(1); + if (snprintf(buf, HUGE_STRING_LENGTH, "intcol%d INT(32), INDEX(intcol%d)", + col_count, col_count) > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in create table\n"); + exit(1); + } + } + else + { + if (snprintf(buf, HUGE_STRING_LENGTH, "intcol%d INT(32) ", col_count) + > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in create table\n"); + exit(1); + } } dynstr_append(&table_string, buf); @@ -741,11 +794,24 @@ build_table_string(void) if (num_char_cols) for (col_count= 1; col_count <= num_char_cols; col_count++) { - if (snprintf(buf, HUGE_STRING_LENGTH, "charcol%d VARCHAR(128)", col_count) - > HUGE_STRING_LENGTH) + if (num_char_cols_index) { - fprintf(stderr, "Memory Allocation error in creating table\n"); - exit(1); + if (snprintf(buf, HUGE_STRING_LENGTH, + "charcol%d VARCHAR(128), INDEX(charcol%d) ", + col_count, col_count) > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating table\n"); + exit(1); + } + } + else + { + if (snprintf(buf, HUGE_STRING_LENGTH, "charcol%d VARCHAR(128)", + col_count) > HUGE_STRING_LENGTH) + { + fprintf(stderr, "Memory Allocation error in creating table\n"); + exit(1); + } } dynstr_append(&table_string, buf); @@ -759,6 +825,7 @@ build_table_string(void) ptr->string = (char *)my_malloc(table_string.length+1, MYF(MY_ZEROFILL|MY_FAE|MY_WME)); ptr->length= table_string.length+1; + ptr->type= CREATE_TABLE_TYPE; strmov(ptr->string, table_string.str); dynstr_free(&table_string); DBUG_RETURN(ptr); @@ -774,7 +841,7 @@ static statement * build_update_string(void) { char buf[HUGE_STRING_LENGTH]; - int col_count; + unsigned int col_count; statement *ptr; DYNAMIC_STRING update_string; DBUG_ENTER("build_update_string"); @@ -818,14 +885,7 @@ build_update_string(void) } if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) - { - if (snprintf(buf, HUGE_STRING_LENGTH, " WHERE id = ") > HUGE_STRING_LENGTH) - { - fprintf(stderr, "Memory Allocation error in creating update_string\n"); - exit(1); - } - dynstr_append(&update_string, buf); - } + dynstr_append(&update_string, " WHERE id = "); ptr= (statement *)my_malloc(sizeof(statement), @@ -854,7 +914,7 @@ static statement * build_insert_string(void) { char buf[HUGE_STRING_LENGTH]; - int col_count; + unsigned int col_count; statement *ptr; DYNAMIC_STRING insert_string; DBUG_ENTER("build_insert_string"); @@ -865,12 +925,7 @@ build_insert_string(void) if (auto_generate_sql_autoincrement) { - if (snprintf(buf, HUGE_STRING_LENGTH, "NULL") > HUGE_STRING_LENGTH) - { - fprintf(stderr, "Memory Allocation error in creating insert\n"); - exit(1); - } - dynstr_append(&insert_string, buf); + dynstr_append(&insert_string, "NULL"); if (num_int_cols || num_char_cols) dynstr_append(&insert_string, ","); @@ -878,12 +933,23 @@ build_insert_string(void) if (auto_generate_sql_guid_primary) { - if (snprintf(buf, HUGE_STRING_LENGTH, "uuid()") > HUGE_STRING_LENGTH) + dynstr_append(&insert_string, "uuid()"); + + if (num_int_cols || num_char_cols) + dynstr_append(&insert_string, ","); + } + + if (auto_generate_sql_secondary_indexes) + { + unsigned int count; + + for (count= 0; count < auto_generate_sql_secondary_indexes; count++) { - fprintf(stderr, "Memory Allocation error in create table\n"); - exit(1); + if (count) /* Except for the first pass we add a comma */ + dynstr_append(&insert_string, ","); + + dynstr_append(&insert_string, "uuid()"); } - dynstr_append(&insert_string, buf); if (num_int_cols || num_char_cols) dynstr_append(&insert_string, ","); @@ -939,7 +1005,7 @@ static statement * build_select_string(my_bool key) { char buf[HUGE_STRING_LENGTH]; - int col_count; + unsigned int col_count; statement *ptr; static DYNAMIC_STRING query_string; DBUG_ENTER("build_select_string"); @@ -1088,6 +1154,29 @@ get_options(int *argc,char ***argv) if (opt_only_print) opt_silent= TRUE; + if (num_int_cols_opt) + { + option_string *str; + parse_option(num_int_cols_opt, &str, ','); + num_int_cols= atoi(str->string); + if (str->option) + num_int_cols_index= atoi(str->option); + option_cleanup(str); + } + + if (num_char_cols_opt) + { + option_string *str; + parse_option(num_char_cols_opt, &str, ','); + num_char_cols= atoi(str->string); + if (str->option) + num_char_cols_index= atoi(str->option); + else + num_char_cols_index= 0; + option_cleanup(str); + } + + if (auto_generate_sql) { unsigned long long x= 0; @@ -1253,7 +1342,7 @@ get_options(int *argc,char ***argv) printf("Parsing engines to use.\n"); if (default_engine) - parse_delimiter(default_engine, &engine_statements, ','); + parse_option(default_engine, &engine_options, ','); if (tty_password) opt_password= get_tty_password(NullS); @@ -1276,10 +1365,8 @@ static int run_query(MYSQL *mysql, const char *query, int len) static int -generate_primary_key_list(MYSQL *mysql, statement *engine_stmt) +generate_primary_key_list(MYSQL *mysql, option_string *engine_stmt) { - char query[HUGE_STRING_LENGTH]; - int len; MYSQL_RES *result; MYSQL_ROW row; unsigned long long counter; @@ -1301,9 +1388,7 @@ generate_primary_key_list(MYSQL *mysql, statement *engine_stmt) } else { - len= snprintf(query, HUGE_STRING_LENGTH, "SELECT id from t1"); - - if (run_query(mysql, query, len)) + if (run_query(mysql, "SELECT id from t1", strlen("SELECT id from t1"))) { fprintf(stderr,"%s: Cannot select GUID primary keys. (%s)\n", my_progname, mysql_error(mysql)); @@ -1352,7 +1437,7 @@ drop_primary_key_list(void) static int create_schema(MYSQL *mysql, const char *db, statement *stmt, - statement *engine_stmt) + option_string *engine_stmt) { char query[HUGE_STRING_LENGTH]; statement *ptr; @@ -1411,11 +1496,27 @@ limit_not_met: if (auto_generate_sql && ( auto_generate_sql_number == count)) break; - if (run_query(mysql, ptr->string, ptr->length)) + if (engine_stmt && engine_stmt->option && ptr->type == CREATE_TABLE_TYPE) { - fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", - my_progname, (uint)ptr->length, ptr->string, mysql_error(mysql)); - exit(1); + char buffer[HUGE_STRING_LENGTH]; + + snprintf(buffer, HUGE_STRING_LENGTH, "%s %s", ptr->string, + engine_stmt->option); + if (run_query(mysql, buffer, strlen(buffer))) + { + fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", + my_progname, (uint)ptr->length, ptr->string, mysql_error(mysql)); + exit(1); + } + } + else + { + if (run_query(mysql, ptr->string, ptr->length)) + { + fprintf(stderr,"%s: Cannot run query %.*s ERROR : %s\n", + my_progname, (uint)ptr->length, ptr->string, mysql_error(mysql)); + exit(1); + } } } @@ -1734,6 +1835,84 @@ end: DBUG_RETURN(0); } +uint +parse_option(const char *origin, option_string **stmt, char delm) +{ + char *retstr; + char *ptr= (char *)origin; + option_string **sptr= stmt; + option_string *tmp; + uint length= strlen(origin); + uint count= 0; /* We know that there is always one */ + + for (tmp= *sptr= (option_string *)my_malloc(sizeof(option_string), + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); + (retstr= strchr(ptr, delm)); + tmp->next= (option_string *)my_malloc(sizeof(option_string), + MYF(MY_ZEROFILL|MY_FAE|MY_WME)), + tmp= tmp->next) + { + char buffer[HUGE_STRING_LENGTH]; + char *buffer_ptr; + + count++; + strncpy(buffer, ptr, (size_t)(retstr - ptr)); + if ((buffer_ptr= strchr(buffer, ':'))) + { + char *option_ptr; + + tmp->length= (size_t)(buffer_ptr - buffer); + tmp->string= my_strndup(ptr, tmp->length, MYF(MY_FAE)); + + option_ptr= ptr + 1 + tmp->length; + + /* Move past the : and the first string */ + tmp->option_length= (size_t)(retstr - option_ptr); + tmp->option= my_strndup(option_ptr, tmp->option_length, + MYF(MY_FAE)); + } + else + { + tmp->string= my_strndup(ptr, (size_t)(retstr - ptr), MYF(MY_FAE)); + tmp->length= (size_t)(retstr - ptr); + } + + ptr+= retstr - ptr + 1; + if (isspace(*ptr)) + ptr++; + count++; + } + + if (ptr != origin+length) + { + char *origin_ptr; + + if ((origin_ptr= strchr(ptr, ':'))) + { + char *option_ptr; + + tmp->length= (size_t)(origin_ptr - ptr); + tmp->string= my_strndup(origin, tmp->length, MYF(MY_FAE)); + + option_ptr= (char *)ptr + 1 + tmp->length; + + /* Move past the : and the first string */ + tmp->option_length= (size_t)((ptr + length) - option_ptr); + tmp->option= my_strndup(option_ptr, tmp->option_length, + MYF(MY_FAE)); + } + else + { + tmp->length= (size_t)((ptr + length) - ptr); + tmp->string= my_strndup(ptr, tmp->length, MYF(MY_FAE)); + } + + count++; + } + + return count; +} + uint parse_delimiter(const char *script, statement **stmt, char delm) @@ -1821,9 +2000,11 @@ void print_conclusions_csv(conclusions *con) { char buffer[HUGE_STRING_LENGTH]; + const char *ptr= auto_generate_sql_type ? auto_generate_sql_type : "query"; snprintf(buffer, HUGE_STRING_LENGTH, - "%s,query,%ld.%03ld,%ld.%03ld,%ld.%03ld,%d,%llu\n", + "%s,%s,%ld.%03ld,%ld.%03ld,%ld.%03ld,%d,%llu\n", con->engine ? con->engine : "", /* Storage engine we ran against */ + ptr, /* Load type */ con->avg_timing / 1000, con->avg_timing % 1000, /* Time to load */ con->min_timing / 1000, con->min_timing % 1000, /* Min time */ con->max_timing / 1000, con->max_timing % 1000, /* Max time */ @@ -1834,7 +2015,7 @@ print_conclusions_csv(conclusions *con) } void -generate_stats(conclusions *con, statement *eng, stats *sptr) +generate_stats(conclusions *con, option_string *eng, stats *sptr) { stats *ptr; unsigned int x; @@ -1866,6 +2047,24 @@ generate_stats(conclusions *con, statement *eng, stats *sptr) con->engine= NULL; } +void +option_cleanup(option_string *stmt) +{ + option_string *ptr, *nptr; + if (!stmt) + return; + + for (ptr= stmt; ptr; ptr= nptr) + { + nptr= ptr->next; + if (ptr->string) + my_free((gptr)ptr->string, MYF(0)); + if (ptr->option) + my_free((gptr)ptr->option, MYF(0)); + my_free((gptr)(byte *)ptr, MYF(0)); + } +} + void statement_cleanup(statement *stmt) { diff --git a/mysql-test/t/mysqlslap.test b/mysql-test/t/mysqlslap.test index 5d98dac03ea..f2bb9a72ecf 100644 --- a/mysql-test/t/mysqlslap.test +++ b/mysql-test/t/mysqlslap.test @@ -32,3 +32,5 @@ --exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --auto-generate-sql-load-type=update --auto-generate-sql-execute-number=5 --exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --auto-generate-sql-load-type=key --auto-generate-sql-execute-number=5 + +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=1 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --auto-generate-sql-guid-primary --auto-generate-sql-load-type=key --auto-generate-sql-execute-number=5 --auto-generate-sql-secondary-indexes=3 diff --git a/sql/authors.h b/sql/authors.h index 48b807c7884..dfe3b143e2f 100644 --- a/sql/authors.h +++ b/sql/authors.h @@ -66,6 +66,7 @@ struct show_table_authors_st show_table_authors[]= { "Parser, port to OS/2, storage engines and some random stuff" }, { "Yuri Dario", "", "OS/2 port" }, { "Andrei Elkin", "Espoo, Finland", "Replication" }, + { "Patrick Galbraith", "Sharon, NH", "Federated Engine, mysqlslap" }, { "Sergei Golubchik", "Kerpen, Germany", "Full-text search, precision math" }, { "Lenz Grimmer", "Hamburg, Germany", From 9055eaa1ebe515530902968e28c404a9a6122cab Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 16 Mar 2007 15:20:22 -0700 Subject: [PATCH 06/11] The pthread() support was not working. Once I fixed use-thread in a previous push I realized that the pthread/windows code was not working. I've replaced the fork/thread design with a pure pthread design using condition timers and broadcast. Ramification, UNIX now uses thread, support for slaves had to be dropped and there is no need for the --use-threads flag. Added --concurrency=0 option so that it will start at 1 and keep going up until something bad happens :) client/client_priv.h: Dead option removed client/mysqlslap.c: Removed lock code, replaced with posix thread code. mysql-test/mysql-test-run.pl: Removed dead option mysql-test/t/mysqlslap.test: Removed dead option --- client/client_priv.h | 1 - client/mysqlslap.c | 355 ++++++++++++++--------------------- mysql-test/mysql-test-run.pl | 3 +- mysql-test/t/mysqlslap.test | 2 +- 4 files changed, 144 insertions(+), 217 deletions(-) diff --git a/client/client_priv.h b/client/client_priv.h index b6543870528..6227dcdee44 100644 --- a/client/client_priv.h +++ b/client/client_priv.h @@ -49,7 +49,6 @@ enum options_client OPT_TRIGGERS, OPT_MYSQL_ONLY_PRINT, OPT_MYSQL_LOCK_DIRECTORY, - OPT_MYSQL_SLAP_SLAVE, OPT_USE_THREADS, OPT_IMPORT_USE_THREADS, OPT_MYSQL_NUMBER_OF_QUERY, diff --git a/client/mysqlslap.c b/client/mysqlslap.c index 0010d8e46ae..1b4692848b9 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -62,7 +62,6 @@ TODO: Add language for better tests String length for files and those put on the command line are not setup to handle binary data. - Report results of each thread into the lock file we use. More stats Break up tests and run them on multiple hosts at once. Allow output to be fed into a database directly. @@ -83,9 +82,7 @@ TODO: #define SELECT_TYPE_REQUIRES_PREFIX 5 #include "client_priv.h" -#ifdef HAVE_LIBPTHREAD #include -#endif #include #include #include @@ -100,9 +97,6 @@ TODO: #endif #include -#define MYSLAPLOCK "/myslaplock.lck" -#define MYSLAPLOCK_DIR "/tmp" - #ifdef __WIN__ #define srandom srand #define random rand @@ -113,6 +107,14 @@ TODO: static char *shared_memory_base_name=0; #endif +/* Global Thread counter */ +uint thread_counter; +pthread_mutex_t counter_mutex; +pthread_cond_t count_threshhold; +uint master_wakeup; +pthread_mutex_t sleeper_mutex; +pthread_cond_t sleep_threshhold; + static char **defaults_argv; char **primary_keys; @@ -127,15 +129,10 @@ const char *delimiter= "\n"; const char *create_schema_string= "mysqlslap"; -const char *lock_directory; -char lock_file_str[FN_REFLEN]; - static my_bool opt_preserve; static my_bool opt_only_print= FALSE; -static my_bool opt_slave; - static my_bool opt_compress= FALSE, tty_password= FALSE, opt_silent= FALSE, auto_generate_sql_autoincrement= FALSE, @@ -175,7 +172,6 @@ static uint opt_protocol= 0; static int get_options(int *argc,char ***argv); static uint opt_mysql_port= 0; -static my_bool opt_use_threads; static const char *load_default_groups[]= { "mysqlslap","client",0 }; @@ -213,7 +209,6 @@ typedef struct thread_context thread_context; struct thread_context { statement *stmt; ulonglong limit; - bool thread; }; typedef struct conclusions conclusions; @@ -256,6 +251,7 @@ static int run_scheduler(stats *sptr, statement *stmts, uint concur, int run_task(thread_context *con); void statement_cleanup(statement *stmt); void option_cleanup(option_string *stmt); +void concurrency_loop(MYSQL *mysql, uint current, option_string *eptr); static const char ALPHANUMERICS[]= "0123456789ABCDEFGHIJKLMNOPQRSTWXYZabcdefghijklmnopqrstuvwxyz"; @@ -289,14 +285,8 @@ static int gettimeofday(struct timeval *tp, void *tzp) int main(int argc, char **argv) { MYSQL mysql; - unsigned int x; - unsigned long long client_limit; option_string *eptr; -#ifdef __WIN__ - opt_use_threads= 1; -#endif - MY_INIT(argv[0]); load_defaults("my",load_default_groups,&argc,&argv); @@ -352,75 +342,33 @@ int main(int argc, char **argv) } } + VOID(pthread_mutex_init(&counter_mutex, NULL)); + VOID(pthread_cond_init(&count_threshhold, NULL)); + VOID(pthread_mutex_init(&sleeper_mutex, NULL)); + VOID(pthread_cond_init(&sleep_threshhold, NULL)); + /* Main iterations loop */ eptr= engine_options; do { /* For the final stage we run whatever queries we were asked to run */ uint *current; - conclusions conclusion; if (verbose >= 2) printf("Starting Concurrency Test\n"); - for (current= concurrency; current && *current; current++) + if (*concurrency) { - stats *head_sptr; - stats *sptr; - - head_sptr= (stats *)my_malloc(sizeof(stats) * iterations, - MYF(MY_ZEROFILL|MY_FAE|MY_WME)); - - bzero(&conclusion, sizeof(conclusions)); - - if (auto_actual_queries) - client_limit= auto_actual_queries; - else if (num_of_query) - client_limit= num_of_query / *current; - else - client_limit= actual_queries; - - for (x= 0, sptr= head_sptr; x < iterations; x++, sptr++) - { - /* - We might not want to load any data, such as when we are calling - a stored_procedure that doesn't use data, or we know we already have - data in the table. - */ - if (!opt_preserve) - drop_schema(&mysql, create_schema_string); - - /* First we create */ - if (create_statements) - create_schema(&mysql, create_schema_string, create_statements, eptr); - - /* - If we generated GUID we need to build a list of them from creation that - we can later use. - */ - if (verbose >= 2) - printf("Generating primary key list\n"); - if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) - generate_primary_key_list(&mysql, eptr); - - run_scheduler(sptr, query_statements, *current, client_limit); - - /* We are finished with this run */ - if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) - drop_primary_key_list(); + for (current= concurrency; current && *current; current++) + concurrency_loop(&mysql, *current, eptr); + } + else + { + uint infinite= 1; + do { + concurrency_loop(&mysql, infinite, eptr); } - - if (verbose >= 2) - printf("Generating stats\n"); - - generate_stats(&conclusion, eptr, head_sptr); - - if (!opt_silent) - print_conclusions(&conclusion); - if (opt_csv_str) - print_conclusions_csv(&conclusion); - - my_free((gptr)head_sptr, MYF(0)); + while (infinite++); } if (!opt_preserve) @@ -428,13 +376,14 @@ int main(int argc, char **argv) } while (eptr ? (eptr= eptr->next) : 0); + VOID(pthread_mutex_destroy(&counter_mutex)); + VOID(pthread_cond_destroy(&count_threshhold)); + VOID(pthread_mutex_destroy(&sleeper_mutex)); + VOID(pthread_cond_destroy(&sleep_threshhold)); + if (!opt_only_print) mysql_close(&mysql); /* Close & free connection */ - - /* Remove lock file */ - my_delete(lock_file_str, MYF(0)); - /* now free all the strings we created */ if (opt_password) my_free((gptr)opt_password, MYF(0)); @@ -455,6 +404,70 @@ int main(int argc, char **argv) return 0; } +void concurrency_loop(MYSQL *mysql, uint current, option_string *eptr) +{ + unsigned int x; + stats *head_sptr; + stats *sptr; + conclusions conclusion; + unsigned long long client_limit; + + head_sptr= (stats *)my_malloc(sizeof(stats) * iterations, + MYF(MY_ZEROFILL|MY_FAE|MY_WME)); + + bzero(&conclusion, sizeof(conclusions)); + + if (auto_actual_queries) + client_limit= auto_actual_queries; + else if (num_of_query) + client_limit= num_of_query / current; + else + client_limit= actual_queries; + + for (x= 0, sptr= head_sptr; x < iterations; x++, sptr++) + { + /* + We might not want to load any data, such as when we are calling + a stored_procedure that doesn't use data, or we know we already have + data in the table. + */ + if (!opt_preserve) + drop_schema(mysql, create_schema_string); + + /* First we create */ + if (create_statements) + create_schema(mysql, create_schema_string, create_statements, eptr); + + /* + If we generated GUID we need to build a list of them from creation that + we can later use. + */ + if (verbose >= 2) + printf("Generating primary key list\n"); + if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) + generate_primary_key_list(mysql, eptr); + + run_scheduler(sptr, query_statements, current, client_limit); + + /* We are finished with this run */ + if (auto_generate_sql_autoincrement || auto_generate_sql_guid_primary) + drop_primary_key_list(); + } + + if (verbose >= 2) + printf("Generating stats\n"); + + generate_stats(&conclusion, eptr, head_sptr); + + if (!opt_silent) + print_conclusions(&conclusion); + if (opt_csv_str) + print_conclusions_csv(&conclusion); + + my_free((gptr)head_sptr, MYF(0)); + +} + static struct my_option my_long_options[] = { @@ -534,9 +547,6 @@ static struct my_option my_long_options[] = REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"iterations", 'i', "Number of times too run the tests.", (gptr*) &iterations, (gptr*) &iterations, 0, GET_UINT, REQUIRED_ARG, 1, 0, 0, 0, 0, 0}, - {"lock-directory", OPT_MYSQL_LOCK_DIRECTORY, "Directory to use to keep locks.", - (gptr*) &lock_directory, (gptr*) &lock_directory, 0, GET_STR, - REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"number-char-cols", 'x', "Number of VARCHAR columns to create table with if specifying --auto-generate-sql ", (gptr*) &num_char_cols_opt, (gptr*) &num_char_cols_opt, 0, GET_STR, REQUIRED_ARG, @@ -584,17 +594,10 @@ static struct my_option my_long_options[] = {"silent", 's', "Run program in silent mode - no output.", (gptr*) &opt_silent, (gptr*) &opt_silent, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, - {"slave", OPT_MYSQL_SLAP_SLAVE, "Follow master locks for other slap clients", - (gptr*) &opt_slave, (gptr*) &opt_slave, 0, GET_BOOL, NO_ARG, - 0, 0, 0, 0, 0, 0}, {"socket", 'S', "Socket file to use for connection.", (gptr*) &opt_mysql_unix_port, (gptr*) &opt_mysql_unix_port, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, #include - {"use-threads", OPT_USE_THREADS, - "Use pthread calls instead of fork() calls (default on Windows)", - (gptr*) &opt_use_threads, (gptr*) &opt_use_threads, 0, - GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, #ifndef DONT_ALLOW_USER_CHANGE {"user", 'u', "User for login if not current user.", (gptr*) &user, (gptr*) &user, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -1126,11 +1129,6 @@ get_options(int *argc,char ***argv) parse_comma(concurrency_str ? concurrency_str : "1", &concurrency); - if (lock_directory) - snprintf(lock_file_str, FN_REFLEN, "%s/%s", lock_directory, MYSLAPLOCK); - else - snprintf(lock_file_str, FN_REFLEN, "%s/%s", MYSLAPLOCK_DIR, MYSLAPLOCK); - if (opt_csv_str) { opt_silent= TRUE; @@ -1553,136 +1551,63 @@ drop_schema(MYSQL *mysql, const char *db) static int run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) { -#ifndef __WIN__ uint x; -#endif - File lock_file; struct timeval start_time, end_time; thread_context con; DBUG_ENTER("run_scheduler"); con.stmt= stmts; con.limit= limit; - con.thread= opt_use_threads ? 1 :0; - lock_file= my_open(lock_file_str, O_CREAT|O_WRONLY|O_TRUNC, MYF(0)); + pthread_t mainthread; /* Thread descriptor */ + pthread_attr_t attr; /* Thread attributes */ - if (!opt_slave) - if (my_lock(lock_file, F_WRLCK, 0, F_TO_EOF, MYF(0))) + pthread_mutex_lock(&counter_mutex); + thread_counter= 0; + + pthread_mutex_lock(&sleeper_mutex); + master_wakeup= 1; + pthread_mutex_unlock(&sleeper_mutex); + for (x= 0; x < concur; x++) + { + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, + PTHREAD_CREATE_DETACHED); + + /* now create the thread */ + if (pthread_create(&mainthread, &attr, (void *)run_task, + (void *)&con) != 0) { - fprintf(stderr,"%s: Could not get lockfile\n", + fprintf(stderr,"%s: Could not create thread\n", my_progname); exit(0); } - -#ifdef HAVE_LIBPTHREAD - if (opt_use_threads) - { - pthread_t mainthread; /* Thread descriptor */ - pthread_attr_t attr; /* Thread attributes */ - - for (x= 0; x < concur; x++) - { - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, - PTHREAD_CREATE_DETACHED); - - /* now create the thread */ - if (pthread_create(&mainthread, &attr, (void *)run_task, - (void *)&con) != 0) - { - fprintf(stderr,"%s: Could not create thread\n", - my_progname); - exit(0); - } - } + thread_counter++; } -#endif -#if !(defined(__WIN__) || defined(__NETWARE__)) -#ifdef HAVE_LIBPTHREAD - else -#endif - { - fflush(NULL); - for (x= 0; x < concur; x++) - { - int pid; - DBUG_PRINT("info", ("x: %d concurrency: %u", x, *concurrency)); - pid= fork(); - switch(pid) - { - case 0: - /* child */ - DBUG_PRINT("info", ("fork returned 0, calling task(\"%s\"), pid %d gid %d", - stmts ? stmts->string : "", pid, getgid())); - if (verbose >= 3) - printf("%s: fork returned 0, calling task pid %d gid %d\n", - my_progname, pid, getgid()); - run_task(&con); - exit(0); - break; - case -1: - /* error */ - DBUG_PRINT("info", - ("fork returned -1, failing pid %d gid %d", pid, getgid())); - fprintf(stderr, - "%s: Failed on fork: -1, max procs per parent exceeded.\n", - my_progname); - /*exit(1);*/ - goto WAIT; - default: - /* parent, forked */ - DBUG_PRINT("info", ("default, break: pid %d gid %d", pid, getgid())); - if (verbose >= 3) - printf("%s: fork returned %d, gid %d\n", - my_progname, pid, getgid()); - break; - } - } - } -#endif + pthread_mutex_unlock(&counter_mutex); - /* Lets release use some clients! */ - if (!opt_slave) - my_lock(lock_file, F_UNLCK, 0, F_TO_EOF, MYF(0)); + pthread_mutex_lock(&sleeper_mutex); + master_wakeup= 0; + pthread_mutex_unlock(&sleeper_mutex); + pthread_cond_broadcast(&sleep_threshhold); gettimeofday(&start_time, NULL); /* - We look to grab a write lock at this point. Once we get it we know that - all clients have completed their work. + We loop until we know that all children have cleaned up. */ - if (opt_use_threads) + pthread_mutex_lock(&counter_mutex); + while (thread_counter) { - if (my_lock(lock_file, F_WRLCK, 0, F_TO_EOF, MYF(0))) - { - fprintf(stderr,"%s: Could not get lockfile\n", - my_progname); - exit(0); - } - my_lock(lock_file, F_UNLCK, 0, F_TO_EOF, MYF(0)); + struct timespec abstime; + + set_timespec(abstime, 3); + pthread_cond_timedwait(&count_threshhold, &counter_mutex, &abstime); } -#ifndef __WIN__ - else - { -WAIT: - while (x--) - { - int status, pid; - pid= wait(&status); - DBUG_PRINT("info", ("Parent: child %d status %d", pid, status)); - if (status != 0) - { - printf("%s: Child %d died with the status %d\n", - my_progname, pid, status); - exit(0); - } - } - } -#endif + pthread_mutex_unlock(&counter_mutex); + gettimeofday(&end_time, NULL); - my_close(lock_file, MYF(0)); sptr->timing= timedif(end_time, start_time); sptr->users= concur; @@ -1696,7 +1621,6 @@ int run_task(thread_context *con) { ulonglong counter= 0, queries; - File lock_file= -1; MYSQL *mysql; MYSQL_RES *result; MYSQL_ROW row; @@ -1705,6 +1629,13 @@ run_task(thread_context *con) DBUG_ENTER("run_task"); DBUG_PRINT("info", ("task script \"%s\"", con->stmt ? con->stmt->string : "")); + pthread_mutex_lock(&sleeper_mutex); + while (master_wakeup) + { + pthread_cond_wait(&sleep_threshhold, &sleeper_mutex); + } + pthread_mutex_unlock(&sleeper_mutex); + if (!(mysql= mysql_init(NULL))) { fprintf(stderr,"%s: mysql_init() failed ERROR : %s\n", @@ -1712,7 +1643,7 @@ run_task(thread_context *con) exit(0); } - if (con->thread && mysql_thread_init()) + if (mysql_thread_init()) { fprintf(stderr,"%s: mysql_thread_init() failed ERROR : %s\n", my_progname, mysql_error(mysql)); @@ -1720,8 +1651,7 @@ run_task(thread_context *con) } DBUG_PRINT("info", ("trying to connect to host %s as user %s", host, user)); - lock_file= my_open(lock_file_str, O_RDWR, MYF(0)); - my_lock(lock_file, F_RDLCK, 0, F_TO_EOF, MYF(0)); + if (!opt_only_print) { /* Connect to server */ @@ -1821,17 +1751,16 @@ limit_not_met: end: - if (lock_file != -1) - { - my_lock(lock_file, F_UNLCK, 0, F_TO_EOF, MYF(0)); - my_close(lock_file, MYF(0)); - } - if (!opt_only_print) mysql_close(mysql); - if (con->thread) - my_thread_end(); + my_thread_end(); + + pthread_mutex_lock(&counter_mutex); + thread_counter--; + pthread_cond_signal(&count_threshhold); + pthread_mutex_unlock(&counter_mutex); + DBUG_RETURN(0); } diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 3e51bee7c8a..ab40f050aa7 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -1877,8 +1877,7 @@ sub environment_setup () { mtr_native_path($exe_mysqlslap) . " -uroot " . "--port=$master->[0]->{'port'} " . - "--socket=$master->[0]->{'path_sock'} --password= " . - "--lock-directory=$opt_tmpdir"; + "--socket=$master->[0]->{'path_sock'} --password= "; if ( $opt_debug ) { diff --git a/mysql-test/t/mysqlslap.test b/mysql-test/t/mysqlslap.test index f2bb9a72ecf..a40a01f1f25 100644 --- a/mysql-test/t/mysqlslap.test +++ b/mysql-test/t/mysqlslap.test @@ -4,7 +4,7 @@ --exec $MYSQL_SLAP --silent --concurrency=5 --iterations=20 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql ---exec $MYSQL_SLAP --silent --concurrency=5 --iterations=20 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --use-threads +--exec $MYSQL_SLAP --silent --concurrency=5 --iterations=20 --number-int-cols=2 --number-char-cols=3 --auto-generate-sql --exec $MYSQL_SLAP --only-print --iterations=20 --query="select * from t1" --create="CREATE TABLE t1 (id int, name varchar(64)); INSERT INTO t1 VALUES (1, 'This is a test')" --delimiter=";" --exec $MYSQL_SLAP --silent --concurrency=5 --iterations=20 --query="select * from t1" --create="CREATE TABLE t1 (id int, name varchar(64)); INSERT INTO t1 VALUES (1, 'This is a test')" --delimiter=";" From c554f58ca15e61cf23a5854d6c653426b29f5961 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 16 Mar 2007 19:05:11 -0700 Subject: [PATCH 07/11] Cleanup of prototype for windows. client/mysqlslap.c: Fixed a prototype for windows. Corrected the call the attr --- client/mysqlslap.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/client/mysqlslap.c b/client/mysqlslap.c index 1b4692848b9..62ee2b03359 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -248,7 +248,7 @@ static int create_schema(MYSQL *mysql, const char *db, statement *stmt, option_string *engine_stmt); static int run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit); -int run_task(thread_context *con); +pthread_handler_t run_task(void *p); void statement_cleanup(statement *stmt); void option_cleanup(option_string *stmt); void concurrency_loop(MYSQL *mysql, uint current, option_string *eptr); @@ -1562,6 +1562,11 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) pthread_t mainthread; /* Thread descriptor */ pthread_attr_t attr; /* Thread attributes */ + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, + PTHREAD_CREATE_DETACHED); + + pthread_mutex_lock(&counter_mutex); thread_counter= 0; @@ -1570,12 +1575,8 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) pthread_mutex_unlock(&sleeper_mutex); for (x= 0; x < concur; x++) { - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, - PTHREAD_CREATE_DETACHED); - - /* now create the thread */ - if (pthread_create(&mainthread, &attr, (void *)run_task, + /* nowucreate the thread */ + if (pthread_create(&mainthread, &attr, run_task, (void *)&con) != 0) { fprintf(stderr,"%s: Could not create thread\n", @@ -1605,6 +1606,7 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) pthread_cond_timedwait(&count_threshhold, &counter_mutex, &abstime); } pthread_mutex_unlock(&counter_mutex); + pthread_attr_destroy(&attr); gettimeofday(&end_time, NULL); @@ -1617,14 +1619,14 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) } -int -run_task(thread_context *con) +pthread_handler_t run_task(void *p) { ulonglong counter= 0, queries; MYSQL *mysql; MYSQL_RES *result; MYSQL_ROW row; statement *ptr; + thread_context *con= (thread_context *)p; DBUG_ENTER("run_task"); DBUG_PRINT("info", ("task script \"%s\"", con->stmt ? con->stmt->string : "")); From d5915675f67ce5beb4f6118ff8d121da36d19b52 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 16 Mar 2007 23:50:48 -0700 Subject: [PATCH 08/11] Bug in windows where attr was not working. Removed. client/mysqlslap.c: attr issue on windows --- client/mysqlslap.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/client/mysqlslap.c b/client/mysqlslap.c index 62ee2b03359..de7a726747e 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -1560,12 +1560,6 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) con.limit= limit; pthread_t mainthread; /* Thread descriptor */ - pthread_attr_t attr; /* Thread attributes */ - - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, - PTHREAD_CREATE_DETACHED); - pthread_mutex_lock(&counter_mutex); thread_counter= 0; @@ -1576,7 +1570,7 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) for (x= 0; x < concur; x++) { /* nowucreate the thread */ - if (pthread_create(&mainthread, &attr, run_task, + if (pthread_create(&mainthread, NULL, run_task, (void *)&con) != 0) { fprintf(stderr,"%s: Could not create thread\n", @@ -1606,7 +1600,6 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) pthread_cond_timedwait(&count_threshhold, &counter_mutex, &abstime); } pthread_mutex_unlock(&counter_mutex); - pthread_attr_destroy(&attr); gettimeofday(&end_time, NULL); @@ -1793,13 +1786,13 @@ parse_option(const char *origin, option_string **stmt, char delm) char *option_ptr; tmp->length= (size_t)(buffer_ptr - buffer); - tmp->string= my_strndup(ptr, tmp->length, MYF(MY_FAE)); + tmp->string= my_strndup(ptr, (uint)tmp->length, MYF(MY_FAE)); option_ptr= ptr + 1 + tmp->length; /* Move past the : and the first string */ tmp->option_length= (size_t)(retstr - option_ptr); - tmp->option= my_strndup(option_ptr, tmp->option_length, + tmp->option= my_strndup(option_ptr, (uint)tmp->option_length, MYF(MY_FAE)); } else @@ -1863,7 +1856,7 @@ parse_delimiter(const char *script, statement **stmt, char delm) tmp= tmp->next) { count++; - tmp->string= my_strndup(ptr, (size_t)(retstr - ptr), MYF(MY_FAE)); + tmp->string= my_strndup(ptr, (uint)(retstr - ptr), MYF(MY_FAE)); tmp->length= (size_t)(retstr - ptr); ptr+= retstr - ptr + 1; if (isspace(*ptr)) @@ -1873,7 +1866,7 @@ parse_delimiter(const char *script, statement **stmt, char delm) if (ptr != script+length) { - tmp->string= my_strndup(ptr, (size_t)((script + length) - ptr), + tmp->string= my_strndup(ptr, (uint)((script + length) - ptr), MYF(MY_FAE)); tmp->length= (size_t)((script + length) - ptr); count++; @@ -1942,7 +1935,7 @@ print_conclusions_csv(conclusions *con) con->users, /* Children used */ con->avg_rows /* Queries run */ ); - my_write(csv_file, buffer, strlen(buffer), MYF(0)); + my_write(csv_file, buffer, (uint)strlen(buffer), MYF(0)); } void From caa6ddd9a436d3adaf3429f0664c486ec88aa9cd Mon Sep 17 00:00:00 2001 From: unknown Date: Sun, 18 Mar 2007 10:06:28 -0700 Subject: [PATCH 09/11] Restoring attr (aka poking around Windows did not work). client/mysqlslap.c: Restorinng attr --- client/mysqlslap.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/client/mysqlslap.c b/client/mysqlslap.c index de7a726747e..6fb810b6a23 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -1560,6 +1560,10 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) con.limit= limit; pthread_t mainthread; /* Thread descriptor */ + pthread_attr_t attr; /* Thread attributes */ + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, + PTHREAD_CREATE_DETACHED); pthread_mutex_lock(&counter_mutex); thread_counter= 0; @@ -1570,7 +1574,7 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) for (x= 0; x < concur; x++) { /* nowucreate the thread */ - if (pthread_create(&mainthread, NULL, run_task, + if (pthread_create(&mainthread, &attr, run_task, (void *)&con) != 0) { fprintf(stderr,"%s: Could not create thread\n", @@ -1580,6 +1584,7 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) thread_counter++; } pthread_mutex_unlock(&counter_mutex); + pthread_attr_destroy(&attr); pthread_mutex_lock(&sleeper_mutex); master_wakeup= 0; From 2b9fdea8c6eff96b938dfd0a47331a02ca186760 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 22 Mar 2007 00:43:14 -0700 Subject: [PATCH 10/11] Fixes for 5.1-arch repository to compile on Windows. CMakeLists.txt: Moved zlib to be processed before extra as CMake doesn't handle forward references well. client/CMakeLists.txt: Ensure that -DTHREADS is specified for mysqlslap.c client/mysqlslap.c: Removed includes which are already included in client_priv.h Moved variable declarations to be at start of function before any code is emitted as VSC is strict. --- CMakeLists.txt | 2 +- client/CMakeLists.txt | 1 + client/mysqlslap.c | 8 ++------ 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eb4d77db73f..48992cf574b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -131,10 +131,10 @@ ADD_SUBDIRECTORY(dbug) ADD_SUBDIRECTORY(strings) ADD_SUBDIRECTORY(regex) ADD_SUBDIRECTORY(mysys) +ADD_SUBDIRECTORY(zlib) ADD_SUBDIRECTORY(extra/yassl) ADD_SUBDIRECTORY(extra/yassl/taocrypt) ADD_SUBDIRECTORY(extra) -ADD_SUBDIRECTORY(zlib) ADD_SUBDIRECTORY(storage/heap) ADD_SUBDIRECTORY(storage/myisam) ADD_SUBDIRECTORY(storage/myisammrg) diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index d492c49a6b2..ea9e7547354 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -95,6 +95,7 @@ ADD_EXECUTABLE(mysqladmin mysqladmin.cc) TARGET_LINK_LIBRARIES(mysqladmin mysqlclient mysys dbug yassl taocrypt zlib wsock32) ADD_EXECUTABLE(mysqlslap mysqlslap.c) +SET_SOURCE_FILES_PROPERTIES(mysqlslap.c PROPERTIES COMPILE_FLAGS "-DTHREADS") TARGET_LINK_LIBRARIES(mysqlslap mysqlclient mysys yassl taocrypt zlib wsock32 dbug) ADD_EXECUTABLE(echo echo.c) diff --git a/client/mysqlslap.c b/client/mysqlslap.c index 6fb810b6a23..809c1a6b5da 100644 --- a/client/mysqlslap.c +++ b/client/mysqlslap.c @@ -82,10 +82,6 @@ TODO: #define SELECT_TYPE_REQUIRES_PREFIX 5 #include "client_priv.h" -#include -#include -#include -#include #include #include #include @@ -1554,13 +1550,13 @@ run_scheduler(stats *sptr, statement *stmts, uint concur, ulonglong limit) uint x; struct timeval start_time, end_time; thread_context con; + pthread_t mainthread; /* Thread descriptor */ + pthread_attr_t attr; /* Thread attributes */ DBUG_ENTER("run_scheduler"); con.stmt= stmts; con.limit= limit; - pthread_t mainthread; /* Thread descriptor */ - pthread_attr_t attr; /* Thread attributes */ pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); From c8e15cf23bb031096682ff4ebc413a2ff6a3abf3 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 22 Mar 2007 01:59:47 -0700 Subject: [PATCH 11/11] fixes for 5.1-arch repository client/Makefile.am: fix flags to ensure that mysqlslap still compiles on Unix --- client/Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/client/Makefile.am b/client/Makefile.am index 1c908ea715e..4b78ee2b0c9 100644 --- a/client/Makefile.am +++ b/client/Makefile.am @@ -76,6 +76,7 @@ mysqlimport_LDADD = $(CXXLDFLAGS) $(CLIENT_THREAD_LIBS) \ mysqlshow_SOURCES= mysqlshow.c mysqlslap_SOURCES= mysqlslap.c +mysqlslap_CFLAGS= -DTHREAD -UUNDEF_THREADS_HACK mysqlslap_LDADD = $(CXXLDFLAGS) $(CLIENT_THREAD_LIBS) \ @CLIENT_EXTRA_LDFLAGS@ \ $(LIBMYSQLCLIENT_LA) \