Allow for disabling content type

Curl automatically sets the content type of
`application/x-www-form-urlencoded`, whilst this hasn't caused us any
problems in the past, Huawei's S3 implementation does not like it. The
two possible options are to remove the content type header, or use mime
multipart uploads.

Multipart uploads could be implemented eventually, but are only really
needed for very large files (> 4GB). So, this patch adds an option to do
the former, disabling the content type. This is not a default option at
the moment and needs to be set using:

```
ms3_set_option(ms3, MS3_OPT_NO_CONTENT_TYPE, NULL);
```

VERSION.txt

@@ -1 +1 @@
-3.1.2
+3.1.3

configure.ac

@@ -38,7 +38,7 @@ AC_USE_SYSTEM_EXTENSIONS
 AC_CONFIG_HEADERS([config.h:config.in])dnl Keep filename to 8.3 for MS-DOS.
 
 # shared library versioning
-LIBMARIAS3_LIBRARY_VERSION=4:2:1
+LIBMARIAS3_LIBRARY_VERSION=4:2:2
 #                          | | |
 #                   +------+ | +---+
 #                   |        |     |

libmarias3/marias3.h

@@ -93,7 +93,8 @@ enum ms3_set_option_t
   MS3_OPT_USER_DATA,
   MS3_OPT_PORT_NUMBER,
   MS3_OPT_CONNECT_TIMEOUT,
-  MS3_OPT_TIMEOUT
+  MS3_OPT_TIMEOUT,
+  MS3_OPT_NO_CONTENT_TYPE
 };
 
 typedef enum ms3_set_option_t ms3_set_option_t;

src/marias3.c

@@ -206,6 +206,7 @@ ms3_st *ms3_init(const char *s3key, const char *s3secret,
   ms3->curl = curl_easy_init();
   ms3->last_error = NULL;
   ms3->use_http = false;
+  ms3->no_content_type = false;
   ms3->disable_verification = false;
   ms3->first_run = true;
   ms3->path_buffer = ms3_cmalloc(sizeof(char) * 1024);
@@ -576,6 +577,12 @@ uint8_t ms3_set_option(ms3_st *ms3, ms3_set_option_t option, void *value)
       break;
     }
 
+    case MS3_OPT_NO_CONTENT_TYPE:
+    {
+      ms3->no_content_type = ms3->no_content_type ? 0 : 1;
+      break;
+    }
+
     case MS3_OPT_BUFFER_CHUNK_SIZE:
     {
       size_t new_size;

src/request.c

@@ -19,6 +19,7 @@
 
 #include "config.h"
 #include "common.h"
+#include "debug.h"
 #include "sha256.h"
 
 #include <curl/curl.h>
@@ -430,7 +431,6 @@ static uint8_t build_request_headers(CURL *curl, struct curl_slist **head,
   uint8_t i;
   bool has_source = false;
   bool has_token = false;
-  struct curl_slist *current_header;
 
   // Host header
   if (base_domain)
@@ -600,15 +600,16 @@ static uint8_t build_request_headers(CURL *curl, struct curl_slist **head,
     headers = curl_slist_append(headers, headerbuf);
   }
 
-  current_header = headers;
-
-  do
+  if (ms3debug_get())
   {
-    ms3debug("Header: %s", current_header->data);
-  }
-  while ((current_header = current_header->next));
+    struct curl_slist *current_header = headers;
 
-  curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+    do
+    {
+      ms3debug("Header: %s", current_header->data);
+    }
+    while ((current_header = current_header->next));
+  }
 
   switch (method)
   {
@@ -821,6 +822,13 @@ uint8_t execute_request(ms3_st *ms3, command_t cmd, const char *bucket,
     return res;
   }
 
+  if (ms3->no_content_type)
+  {
+    headers = curl_slist_append(headers, "Content-Type:");
+  }
+
+  curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+
   if (ms3->disable_verification)
   {
     ms3debug("Disabling SSL verification");

src/structs.h

@@ -60,6 +60,7 @@ struct ms3_st
   CURL *curl;
   char *last_error;
   bool use_http;
+  bool no_content_type;
   bool disable_verification;
   uint8_t list_version;
   uint8_t protocol_version;

tests/basic_no_type.c

@@ -0,0 +1,180 @@
+/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
+ * Copyright 2019 MariaDB Corporation Ab. All rights reserved.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+ * MA 02110-1301  USA
+ */
+
+#include <yatl/lite.h>
+#include <libmarias3/marias3.h>
+
+/* Tests basic put, list, get, status, delete using the thread calls */
+
+int main(int argc, char *argv[])
+{
+  int res;
+  ms3_list_st *list = NULL, *list_it = NULL;
+  uint8_t *data;
+  size_t length;
+  int i;
+  bool found;
+  uint8_t list_version;
+  const char *test_string = "Another one bites the dust";
+  ms3_status_st status;
+  ms3_st *ms3;
+  char *s3key = getenv("S3KEY");
+  char *s3secret = getenv("S3SECRET");
+  char *s3region = getenv("S3REGION");
+  char *s3bucket = getenv("S3BUCKET");
+  char *s3host = getenv("S3HOST");
+  char *s3noverify = getenv("S3NOVERIFY");
+  char *s3usehttp = getenv("S3USEHTTP");
+  char *s3port = getenv("S3PORT");
+
+  SKIP_IF_(!s3key, "Environemnt variable S3KEY missing");
+  SKIP_IF_(!s3secret, "Environemnt variable S3SECRET missing");
+  SKIP_IF_(!s3region, "Environemnt variable S3REGION missing");
+  SKIP_IF_(!s3bucket, "Environemnt variable S3BUCKET missing");
+
+  (void) argc;
+  (void) argv;
+
+  ms3_library_init();
+  ms3 = ms3_init(s3key, s3secret, s3region, s3host);
+
+  if (s3noverify && !strcmp(s3noverify, "1"))
+  {
+    ms3_set_option(ms3, MS3_OPT_DISABLE_SSL_VERIFY, NULL);
+  }
+
+  if (s3usehttp && !strcmp(s3usehttp, "1"))
+  {
+    ms3_set_option(ms3, MS3_OPT_USE_HTTP, NULL);
+  }
+
+  if (s3port)
+  {
+    int port = atol(s3port);
+    ms3_set_option(ms3, MS3_OPT_PORT_NUMBER, &port);
+  }
+  ms3_set_option(ms3, MS3_OPT_NO_CONTENT_TYPE, NULL);
+//  ms3_debug();
+  ASSERT_NOT_NULL(ms3);
+
+  res = ms3_put(ms3, s3bucket, "test/basic_no_type.txt",
+                (const uint8_t *)test_string,
+                strlen(test_string));
+  ASSERT_EQ_(res, 0, "Result: %u", res);
+
+  // A prefix that will give no results;
+  res = ms3_list(ms3, s3bucket, "asdfghjkl", &list);
+  ASSERT_EQ_(res, 0, "Result: %u", res);
+  ASSERT_NULL_(list, "List not empty");
+
+  res = ms3_list(ms3, s3bucket, NULL, &list);
+  ASSERT_EQ_(res, 0, "Result: %u", res);
+  found = false;
+  list_it = list;
+
+  while (list_it)
+  {
+    if (!strncmp(list_it->key, "test/basic_no_type.txt", 21))
+    {
+      found = true;
+      break;
+    }
+
+    list_it = list_it->next;
+  }
+
+  ASSERT_EQ_(found, 1, "Created file not found");
+
+  if (list_it)
+  {
+    ASSERT_EQ_(list_it->length, 26, "Created file is unexpected length");
+    ASSERT_NEQ_(list_it->created, 0, "Created file timestamp is bad");
+  }
+  else
+  {
+    ASSERT_TRUE_(false, "No resuts from list");
+  }
+
+  // Retry list with V1 API
+  list_version = 1;
+  list = NULL;
+  ms3_set_option(ms3, MS3_OPT_FORCE_LIST_VERSION, &list_version);
+  res = ms3_list(ms3, s3bucket, NULL, &list);
+  ASSERT_EQ_(res, 0, "Result: %u", res);
+  found = false;
+  list_it = list;
+
+  while (list_it)
+  {
+    if (!strncmp(list_it->key, "test/basic_no_type.txt", 21))
+    {
+      found = true;
+      break;
+    }
+
+    list_it = list_it->next;
+  }
+
+  ASSERT_EQ_(found, 1, "Created file not found");
+
+  if (list_it)
+  {
+    ASSERT_EQ_(list_it->length, 26, "Created file is unexpected length");
+    ASSERT_NEQ_(list_it->created, 0, "Created file timestamp is bad");
+  }
+  else
+  {
+    ASSERT_TRUE_(false, "No resuts from list");
+  }
+
+  res = ms3_get(ms3, s3bucket, "test/basic_no_type.txt", &data, &length);
+  ASSERT_EQ_(res, 0, "Result: %u", res);
+  ASSERT_EQ(length, 26);
+  ASSERT_STREQ((char *)data, test_string);
+
+  for (i = 0; i <= 3; i++)
+  {
+    res = ms3_status(ms3, s3bucket, "test/basic_no_type.txt", &status);
+
+    if (res == MS3_ERR_NOT_FOUND)
+    {
+      continue;
+    }
+
+    ASSERT_EQ_(res, 0, "Result: %u", res);
+
+    if (res == 0)
+    {
+      break;
+    }
+  }
+
+  ASSERT_EQ(status.length, 26);
+  ASSERT_NEQ(status.created, 0);
+  res = ms3_delete(ms3, s3bucket, "test/basic_no_type.txt");
+  ASSERT_EQ_(res, 0, "Result: %u", res);
+  ms3_free(data);
+  res = ms3_get(ms3, s3bucket, "test/basic_no_type.txt", &data, &length);
+  ASSERT_NEQ_(res, 0, "Object should error");
+  ASSERT_NULL_(data, "Data should be NULL");
+  ASSERT_EQ_(length, 0, "There should be no data");
+  ms3_deinit(ms3);
+  ms3_library_deinit();
+  return 0;
+}

tests/include.am

@@ -21,6 +21,11 @@ t_basic_LDADD= src/libmarias3.la
 check_PROGRAMS+= t/basic
 noinst_PROGRAMS+= t/basic
 
+t_basic_no_type_SOURCES= tests/basic_no_type.c
+t_basic_no_type_LDADD= src/libmarias3.la
+check_PROGRAMS+= t/basic_no_type
+noinst_PROGRAMS+= t/basic_no_type
+
 t_snowman_SOURCES= tests/snowman.c
 t_snowman_LDADD= src/libmarias3.la
 check_PROGRAMS+= t/snowman