CONC-387 return MYSQL_DATA_TRUNCATED for invalid numeric strings.

Fix a regression in new my_atoll()/my_atoull()
to set error, if there are non-digits found in the string.

Spaces at the start and end of string are ignored (however, not between the
digits).

libmariadb/ma_stmt_codec.c

@@ -209,6 +209,7 @@ static long long my_strtoll(const char *str, size_t len, const char **end, int *
   return -1LL * uval;
 }
 
+
 static long long my_atoll(const char *str, const char *end_str, int *error)
 {
   const char *p=str;
@@ -216,18 +217,36 @@ static long long my_atoll(const char *str, const char *end_str, int *error)
   long long ret;
   while (p < end_str && isspace(*p))
     p++;
+
   ret = my_strtoll(p, end_str - p, &end, error);
+
+  while(end < end_str && isspace(*end))
+   end++;
+
+  if(end != end_str)
+    *error= 1;
+
   return ret;
 }
 
+
 static unsigned long long my_atoull(const char *str, const char *end_str, int *error)
 {
   const char *p = str;
   const char *end;
   unsigned long long ret;
+
   while (p < end_str && isspace(*p))
     p++;
+
   ret = my_strtoull(p, end_str - p, &end, error);
+
+  while(end < end_str && isspace(*end))
+   end++;
+
+  if(end != end_str)
+    *error= 1;
+
   return ret;
 }
 

unittest/libmariadb/ps_bugs.c

@@ -4658,6 +4658,62 @@ static int equal_MYSQL_TIME(MYSQL_TIME *tm1, MYSQL_TIME *tm2)
     tm1->second_part==tm2->second_part && tm1->time_type==tm2->time_type && tm1->year==tm2->year;
 }
 
+static int test_str_to_int(MYSQL *mysql)
+{
+ int i;
+ struct st_atoi_test{
+    const char *str_value;
+    int int_value;
+    int rc;
+  } atoi_tests[]=
+  {
+    {"0", 0, 0},
+    {" 1",1, 0},
+    {"123 ",123, 0},
+    {"10.2",10, MYSQL_DATA_TRUNCATED},
+    {"a", 0, MYSQL_DATA_TRUNCATED},
+    {"1 2 3", 1, MYSQL_DATA_TRUNCATED},
+    {NULL, 0, 0}
+  };
+
+  for(i=0; atoi_tests[i].str_value; i++)
+  {
+    int rc;
+    MYSQL_STMT *stmt;
+    MYSQL_BIND bind[1];
+    struct st_atoi_test *test= &atoi_tests[i];
+    char sql[256];
+    int int_value;
+
+    snprintf(sql, sizeof(sql), "SELECT '%s'",test->str_value);
+
+    stmt= mysql_stmt_init(mysql);
+
+    rc= mysql_stmt_prepare(stmt, sql, strlen(sql));
+    check_stmt_rc(rc, stmt);
+    rc= mysql_stmt_execute(stmt);
+    check_stmt_rc(rc, stmt);
+    rc= mysql_stmt_store_result(stmt);
+
+    memset(bind, 0, sizeof(MYSQL_BIND));
+    bind[0].buffer_type= MYSQL_TYPE_LONG;
+    bind[0].buffer= &int_value;
+    bind[0].buffer_length= sizeof(int_value);
+
+    rc= mysql_stmt_bind_result(stmt, bind);
+    check_stmt_rc(rc, stmt);
+    rc= mysql_stmt_fetch(stmt);
+
+    diag("test: str='%s', expected/returned value =%d/%d, expected/returned rc=%d/%d",
+      test->str_value, test->int_value, int_value, test->rc, rc);
+    FAIL_UNLESS(rc == test->rc, "unexpected return code");
+    FAIL_UNLESS(int_value == test->int_value, "unexpected int value");
+    mysql_stmt_close(stmt);
+  }
+  return OK;
+}
+
+
 static int test_codbc138(MYSQL *mysql)
 {
   int rc;
@@ -5029,6 +5085,7 @@ struct my_tests_st my_tests[] = {
   {"test_stiny_bug", test_stiny_bug, TEST_CONNECTION_DEFAULT, 0, NULL , NULL},
   {"test_bug53311", test_bug53311, TEST_CONNECTION_NEW, 0, NULL , NULL},
   {"test_conc_fraction", test_conc_fraction, TEST_CONNECTION_DEFAULT, 0, NULL , NULL},
+  {"test_str_to_int", test_str_to_int, TEST_CONNECTION_DEFAULT, 0, NULL, NULL},
   {NULL, NULL, 0, 0, NULL, NULL}
 };