Add new XML parser and fix bugs

LibXML has been removed in favour of a modified version of xml.c
(modified to work with AWS responses and our code base usage).

Also fixes:

* AC_MSG_ERROR usage (fixes #83)
* Some places where we weren't using ms3_c alloc functions
* ms3_get doing an ms3_list with an empty key (fixes #82)

README.rst

@@ -55,6 +55,6 @@ The libMariaS3 authors are:
 libMariaS3 uses the following Open Source projects:
 
 * `libcurl <https://curl.haxx.se/>`_
-* `libxml2 <http://www.xmlsoft.org/>`_
+* `xml.c <https://github.com/ooxi/xml.c/>`_
 * `DDM4 <https://github.com/TangentOrg/ddm4>`_
 * `Jouni Malinen's SHA256 hash code <j@w1.fi>`_

configure.ac

@@ -71,7 +71,6 @@ AC_CHECK_PROGS([RPM],[rpm])
 # Checks for typedefs, structures, and compiler characteristics.
 
 PKG_CHECK_MODULES([LIBCURL], [libcurl >= 7.0], [ax_cv_libcurl=yes], [AC_MSG_ERROR(could not find a suitable version of libcurl)])
-PKG_CHECK_MODULES([LIBXML2], [libxml-2.0 >= 2.0], [ax_cv_libxml2=yes], [AC_MSG_ERROR(could not find a suitable version of libxml2)])
 LT_LIB_M
 
 AX_ENDIAN

docs/api/functions.rst

@@ -21,7 +21,7 @@ ms3_library_init_malloc()
 
 .. c:function:: uint8_t ms3_library_init_malloc(ms3_malloc_callback m, ms3_free_callback f, ms3_realloc_callback r, ms3_strdup_callback s, ms3_calloc_callback c)
 
-   Initialize the library for use with custom allocator replacement functions. These functions are also fed into libcurl and libxml2. The function prototypes should be as follows:
+   Initialize the library for use with custom allocator replacement functions. These functions are also fed into libcurl. The function prototypes should be as follows:
 
    .. c:function:: void *ms3_malloc_callback(size_t size)
 

docs/appendix/credits.rst

@@ -10,6 +10,6 @@ The libMariaS3 authors are:
 libMariaS3 uses the following Open Source projects:
 
 * `libcurl <https://curl.haxx.se/>`_
-* `libxml2 <http://www.xmlsoft.org/>`_
+* `xml.c <https://github.com/ooxi/xml.c/>`_
 * `DDM4 <https://github.com/TangentOrg/ddm4>`_
 * `Jouni Malinen's SHA256 hash code <j@w1.fi>`_

docs/appendix/version_history.rst

@@ -9,6 +9,9 @@ Version 3.1.2 GA
 
 * Make library work with quirks in Google Cloud's S3 implementation
 * Detect when libcurl was built with OpenSSL < 1.1.0 and add workaround to thread safety issues in the older OpenSSL versions (affects Ubuntu 16.04 in particular)
+* Remove libxml and replace it with a modified version of `xml.c <https://github.com/ooxi/xml.c>`_ which handles <? ?> tags and other minor changes
+* Fix issue where an empty key for :c:func:`ms3_get` turns it into a list call
+* Partially fix issue with ``AC_MSG_ERROR``. Will still fail if you don't have ``libtool`` and ``pkg-config`` installed.
 
 Version 3.1.1 GA (2019-06-28)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

rpm/libmarias3.spec.in

@@ -4,7 +4,7 @@ Version: @VERSION@
 Release: 1
 License: LGPL v2.1
 Group: System Environment/Libraries
-BuildRequires: libcurl libxml2
+BuildRequires: libcurl
 URL: https://github.com/mariadb-corporation/libmarias3
 
 Packager: Andrew Hutchings <linuxjedi@mariadb.com>

src/include.am

@@ -8,13 +8,13 @@ noinst_HEADERS+= src/error.h
 noinst_HEADERS+= src/structs.h
 noinst_HEADERS+= src/request.h
 noinst_HEADERS+= src/response.h
+noinst_HEADERS+= src/xml.h
 
 lib_LTLIBRARIES+= src/libmarias3.la
 src_libmarias3_la_SOURCES=
 src_libmarias3_la_LIBADD=
 src_libmarias3_la_LDFLAGS=
 src_libmarias3_la_CFLAGS= -DBUILDING_MS3
-src_libmarias3_la_CFLAGS+= @LIBXML2_CFLAGS@
 
 src_libmarias3_la_SOURCES+= src/marias3.c
 src_libmarias3_la_SOURCES+= src/request.c
@@ -25,6 +25,8 @@ src_libmarias3_la_SOURCES+= src/debug.c
 src_libmarias3_la_SOURCES+= src/sha256.c
 src_libmarias3_la_SOURCES+= src/sha256-internal.c
 
+src_libmarias3_la_SOURCES+= src/xml.c
+
 src_libmarias3_la_LDFLAGS+= -version-info ${LIBMARIAS3_LIBRARY_VERSION}
 
-src_libmarias3_la_LIBADD+= @LIBCURL_LIBS@ @LIBXML2_LIBS@ @LIBM@
+src_libmarias3_la_LIBADD+= @LIBCURL_LIBS@ @LIBM@

src/marias3.c

@@ -20,7 +20,6 @@
 #include "config.h"
 #include "common.h"
 
-#include <libxml/xmlmemory.h>
 #include <pthread.h>
 #include <arpa/inet.h>
 
@@ -67,12 +66,19 @@ uint8_t ms3_library_init_malloc(ms3_malloc_callback m,
   ms3_cstrdup = s;
   ms3_ccalloc = c;
 
-  if (curl_global_init_mem(CURL_GLOBAL_DEFAULT, m, f, r, s, c))
+#ifdef HAVE_CURL_OPENSSL_UNSAFE
+  int i;
+  mutex_buf = ms3_cmalloc(CRYPTO_num_locks() * sizeof(pthread_mutex_t));
+  if(mutex_buf)
   {
-    return MS3_ERR_PARAMETER;
+    for(i = 0; i < CRYPTO_num_locks(); i++)
+      pthread_mutex_init(&(mutex_buf[i]), NULL);
+    CRYPTO_set_id_callback(id_function);
+    CRYPTO_set_locking_callback(locking_function);
   }
+#endif
 
-  if (xmlMemSetup(f, m, r, s))
+  if (curl_global_init_mem(CURL_GLOBAL_DEFAULT, m, f, r, s, c))
   {
     return MS3_ERR_PARAMETER;
   }
@@ -94,7 +100,6 @@ void ms3_library_init(void)
   }
 #endif
   curl_global_init(CURL_GLOBAL_DEFAULT);
-  xmlInitParser();
 }
 
 void ms3_library_deinit(void)
@@ -107,12 +112,11 @@ void ms3_library_deinit(void)
     CRYPTO_set_locking_callback(NULL);
     for(i = 0;  i < CRYPTO_num_locks();  i++)
       pthread_mutex_destroy(&(mutex_buf[i]));
-    free(mutex_buf);
+    ms3_cfree(mutex_buf);
     mutex_buf = NULL;
   }
 #endif
   curl_global_cleanup();
-  xmlCleanupParser();
 }
 
 ms3_st *ms3_init(const char *s3key, const char *s3secret,
@@ -184,7 +188,7 @@ static void list_free(ms3_st *ms3)
   struct ms3_pool_alloc_list_st *plist = NULL, *next = NULL;
   while (list)
   {
-    xmlFree(list->key);
+    ms3_cfree(list->key);
     list = list->next;
   }
   plist = ms3->list_container.pool_list;
@@ -327,7 +331,7 @@ uint8_t ms3_get(ms3_st *ms3, const char *bucket, const char *key,
   buf.data = NULL;
   buf.length = 0;
 
-  if (!ms3 || !bucket || !key || !data || !length)
+  if (!ms3 || !bucket || !key || key[0] == '\0' || !data || !length)
   {
     return MS3_ERR_PARAMETER;
   }

src/response.c

@@ -20,51 +20,54 @@
 #include "config.h"
 #include "common.h"
 
-#include <libxml/parser.h>
-#include <libxml/tree.h>
+#include "xml.h"
 
 char *parse_error_message(const char *data, size_t length)
 {
-  xmlDocPtr doc;
-  xmlNodePtr node;
-  xmlChar *message = NULL;
+  struct xml_document *doc = NULL;
+  struct xml_node *node = NULL;
+  struct xml_node *root = NULL;
+
+  uint64_t node_it = 0;
 
   if (!data || !length)
   {
     return NULL;
   }
 
-  doc = xmlReadMemory(data, (int)length, "noname.xml", NULL, 0);
+  doc = xml_parse_document((uint8_t*)data, length);
 
   if (!doc)
   {
     return NULL;
   }
 
-  node = xmlDocGetRootElement(doc);
+  root = xml_document_root(doc);
 
   if (!node)
   {
-    xmlFreeDoc(doc);
+    xml_document_free(doc, false);
     return NULL;
   }
 
   // First node is Error
-  node = node->xmlChildrenNode;
-
-  while (node)
+  node = xml_node_child(root, node_it);
+  while(node)
   {
-    if (!xmlStrcmp(node->name, (const unsigned char *)"Message"))
+    if (!xml_node_name_cmp(node, "Message"))
     {
-      message = xmlNodeGetContent(node);
-      xmlFreeDoc(doc);
+      struct xml_string *content = xml_node_content(node);
+      uint8_t *message = ms3_cmalloc(xml_string_length(content) + 1);
+      xml_string_copy(content, message, xml_string_length(content));
+      xml_document_free(doc, false);
       return (char *)message;
     }
 
-    node = node->next;
+    node_it++;
+    node = xml_node_child(root, node_it);
   }
 
-  xmlFreeDoc(doc);
+  xml_document_free(doc, false);
   return NULL;
 }
 
@@ -117,18 +120,20 @@ uint8_t parse_list_response(const char *data, size_t length, struct ms3_list_con
                             uint8_t list_version,
                             char **continuation)
 {
-  xmlDocPtr doc;
-  xmlNodePtr node;
-  xmlNodePtr child;
-  xmlChar *filename = NULL;
-  xmlChar *filesize = NULL;
-  xmlChar *filedate = NULL;
+  struct xml_document *doc;
+  struct xml_node *root;
+  struct xml_node *node;
+  struct xml_node *child;
+  char *filename = NULL;
+  char *filesize = NULL;
+  char *filedate = NULL;
   size_t size = 0;
   struct tm ttmp = {0};
   time_t tout = 0;
   bool truncated = false;
   const char *last_key = NULL;
   ms3_list_st *nextptr = NULL, *lastptr = list_container->next;
+  uint64_t node_it = 0;
 
   // Empty list
   if (!data || !length)
@@ -136,7 +141,7 @@ uint8_t parse_list_response(const char *data, size_t length, struct ms3_list_con
     return 0;
   }
 
-  doc = xmlReadMemory(data, (int)length, "noname.xml", NULL, 0);
+  doc = xml_parse_document((uint8_t*)data, length);
 
   if (!doc)
   {
@@ -152,77 +157,91 @@ uint8_t parse_list_response(const char *data, size_t length, struct ms3_list_con
    * We use the "continuation" return value for both
    */
 
-  node = xmlDocGetRootElement(doc);
+  root = xml_document_root(doc);
   // First node is ListBucketResponse
-  node = node->xmlChildrenNode;
+  node = xml_node_child(root, 0);
 
   do
   {
-    if (!xmlStrcmp(node->name, (const unsigned char *)"NextContinuationToken"))
+    if (!xml_node_name_cmp(node, "NextContinuationToken"))
     {
-      *continuation = (char *)xmlNodeGetContent(node);
+      struct xml_string *content = xml_node_content(node);
+      *continuation = ms3_cmalloc(xml_string_length(content) + 1);
+      xml_string_copy(content, (uint8_t*)*continuation, xml_string_length(content));
       continue;
     }
 
     if (list_version == 1)
     {
-      if (!xmlStrcmp(node->name, (const unsigned char *)"IsTruncated"))
+      if (!xml_node_name_cmp(node, "IsTruncated"))
       {
-        xmlChar *trunc_value = xmlNodeGetContent(node);
+        struct xml_string *content = xml_node_content(node);
+        char *trunc_value = ms3_cmalloc(xml_string_length(content) + 1);
+        xml_string_copy(content, (uint8_t*)trunc_value, xml_string_length(content));
 
-        if (!xmlStrcmp(trunc_value, (const unsigned char *)"true"))
+        if (!strcmp(trunc_value, "true"))
         {
           truncated = true;
         }
 
-        xmlFree(trunc_value);
+        ms3_cfree(trunc_value);
         continue;
       }
     }
 
-    if (!xmlStrcmp(node->name, (const unsigned char *)"Contents"))
+    if (!xml_node_name_cmp(node, "Contents"))
     {
       bool skip = false;
+      uint64_t child_it = 0;
       // Found contents
-      child = node->xmlChildrenNode;
+      child = xml_node_child(node, 0);
 
       do
       {
-        if (!xmlStrcmp(child->name, (const unsigned char *)"Key"))
+        if (!xml_node_name_cmp(child, "Key"))
         {
-          filename = xmlNodeGetContent(child);
+          struct xml_string *content = xml_node_content(child);
+          filename = ms3_cmalloc(xml_string_length(content) + 1);
+          xml_string_copy(content, (uint8_t*)filename, xml_string_length(content));
+
           ms3debug("Filename: %s", filename);
 
           if (filename[strlen((const char *)filename) - 1] == '/')
           {
             skip = true;
-            xmlFree(filename);
+            ms3_cfree(filename);
             break;
           }
 
           continue;
         }
 
-        if (!xmlStrcmp(child->name, (const unsigned char *)"Size"))
+        if (!xml_node_name_cmp(child, "Size"))
         {
-          filesize = xmlNodeGetContent(child);
+          struct xml_string *content = xml_node_content(child);
+          filesize = ms3_cmalloc(xml_string_length(content) + 1);
+          xml_string_copy(content, (uint8_t*)filesize, xml_string_length(content));
+
           ms3debug("Size: %s", filesize);
           size = strtoull((const char *)filesize, NULL, 10);
-          xmlFree(filesize);
+          ms3_cfree(filesize);
           continue;
         }
 
-        if (!xmlStrcmp(child->name, (const unsigned char *)"LastModified"))
+        if (!xml_node_name_cmp(child, "LastModified"))
         {
-          filedate = xmlNodeGetContent(child);
+          struct xml_string *content = xml_node_content(child);
+          filedate = ms3_cmalloc(xml_string_length(content) + 1);
+          xml_string_copy(content, (uint8_t*)filedate, xml_string_length(content));
+
           ms3debug("Date: %s", filedate);
           strptime((const char *)filedate, "%Y-%m-%dT%H:%M:%SZ", &ttmp);
           tout = mktime(&ttmp);
-          xmlFree(filedate);
+          ms3_cfree(filedate);
           continue;
         }
       }
-      while ((child = child->next));
+      while ((child = xml_node_child(node, ++child_it)));
 
       if (!skip)
       {
@@ -256,13 +275,16 @@ uint8_t parse_list_response(const char *data, size_t length, struct ms3_list_con
       continue;
     }
 
-    if (!xmlStrcmp(node->name, (const unsigned char *)"CommonPrefixes"))
+    if (!xml_node_name_cmp(node, "CommonPrefixes"))
     {
-      child = node->xmlChildrenNode;
+      child = xml_node_child(node, 0);
 
-      if (!xmlStrcmp(child->name, (const unsigned char *)"Prefix"))
+      if (!xml_node_name_cmp(child, "Prefix"))
       {
-        filename = xmlNodeGetContent(child);
+        struct xml_string *content = xml_node_content(child);
+        filename = ms3_cmalloc(xml_string_length(content) + 1);
+        xml_string_copy(content, (uint8_t*)filename, xml_string_length(content));
+
         ms3debug("Filename: %s", filename);
         nextptr = get_next_list_ptr(list_container);
         nextptr->next = NULL;
@@ -280,13 +302,13 @@ uint8_t parse_list_response(const char *data, size_t length, struct ms3_list_con
     }
 
   }
-  while ((node = node->next));
+  while ((node = xml_node_child(root, ++node_it)));
 
   if (list_version == 1 && truncated && last_key)
   {
     *continuation = ms3_cstrdup(last_key);
   }
 
-  xmlFreeDoc(doc);
+  xml_document_free(doc, false);
   return 0;
 }

src/xml.h

@@ -0,0 +1,197 @@
+/**
+ * Copyright (c) 2012 ooxi/xml.c
+ *     https://github.com/ooxi/xml.c
+ *
+ * This software is provided 'as-is', without any express or implied warranty.
+ * In no event will the authors be held liable for any damages arising from the
+ * use of this software.
+ * 
+ * Permission is granted to anyone to use this software for any purpose,
+ * including commercial applications, and to alter it and redistribute it
+ * freely, subject to the following restrictions:
+ *
+ *  1. The origin of this software must not be misrepresented; you must not
+ *     claim that you wrote the original software. If you use this software in a
+ *     product, an acknowledgment in the product documentation would be
+ *     appreciated but is not required.
+ * 
+ *  2. Altered source versions must be plainly marked as such, and must not be
+ *     misrepresented as being the original software.
+ *
+ *  3. This notice may not be removed or altered from any source distribution.
+ */
+#ifndef HEADER_XML
+#define HEADER_XML
+
+
+/**
+ * Includes
+ */
+#include <stdint.h>
+#include <string.h>
+#include <stdbool.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * Opaque structure holding the parsed xml document
+ */
+struct xml_document;
+struct xml_node;
+struct xml_attribute;
+
+/**
+ * Internal character sequence representation
+ */
+struct xml_string;
+
+
+
+/**
+ * Tries to parse the XML fragment in buffer
+ *
+ * @param buffer Chunk to parse
+ * @param length Size of the buffer
+ *
+ * @warning `buffer` will be referenced by the document, you may not free it
+ *     until you free the xml_document
+ * @warning You have to call xml_document_free after you finished using the
+ *     document
+ *
+ * @return The parsed xml fragment iff parsing was successful, 0 otherwise
+ */
+struct xml_document* xml_parse_document(uint8_t* buffer, size_t length);
+
+
+
+/**
+ * Tries to read an XML document from disk
+ *
+ * @param source File that will be read into an xml document. Will be closed
+ *
+ * @warning You have to call xml_document_free with free_buffer = true after you
+ *     finished using the document
+ *
+ * @return The parsed xml fragment iff parsing was successful, 0 otherwise
+ */
+struct xml_document* xml_open_document(FILE* source);
+
+
+
+/**
+ * Frees all resources associated with the document. All xml_node and xml_string
+ * references obtained through the document will be invalidated
+ *
+ * @param document xml_document to free
+ * @param free_buffer iff true the internal buffer supplied via xml_parse_buffer
+ *     will be freed with the `free` system call
+ */
+void xml_document_free(struct xml_document* document, bool free_buffer);
+
+
+/**
+ * @return xml_node representing the document root
+ */
+struct xml_node* xml_document_root(struct xml_document* document);
+
+
+
+/**
+ * @return The xml_node's tag name
+ */
+struct xml_string* xml_node_name(struct xml_node* node);
+
+
+
+/**
+ * @return The xml_node's string content (if available, otherwise NULL)
+ */
+struct xml_string* xml_node_content(struct xml_node* node);
+
+
+
+/**
+ * @return Number of child nodes
+ */
+size_t xml_node_children(struct xml_node* node);
+
+
+
+/**
+ * @return The n-th child or 0 if out of range
+ */
+struct xml_node* xml_node_child(struct xml_node* node, size_t child);
+
+
+
+/**
+ * @return Number of attribute nodes
+ */
+size_t xml_node_attributes(struct xml_node* node);
+
+
+
+/**
+ * @return the n-th attribute name or 0 if out of range
+ */
+struct xml_string* xml_node_attribute_name(struct xml_node* node, size_t attribute);
+
+
+
+/**
+ * @return the n-th attribute content or 0 if out of range
+ */
+struct xml_string* xml_node_attribute_content(struct xml_node* node, size_t attribute);
+
+
+
+/**
+ * @return The node described by the path or 0 if child cannot be found
+ * @warning Each element on the way must be unique
+ * @warning Last argument must be 0
+ */
+struct xml_node* xml_easy_child(struct xml_node* node, uint8_t const* child, ...);
+
+
+
+/**
+ * @return 0-terminated copy of node name
+ * @warning User must free the result
+ */
+uint8_t* xml_easy_name(struct xml_node* node);
+
+
+
+/**
+ * @return 0-terminated copy of node content
+ * @warning User must free the result
+ */
+uint8_t* xml_easy_content(struct xml_node* node);
+
+
+
+/**
+ * @return Length of the string
+ */
+size_t xml_string_length(struct xml_string* string);
+
+
+
+/**
+ * Copies the string into the supplied buffer
+ *
+ * @warning String will not be 0-terminated
+ * @warning Will write at most length bytes, even if the string is longer
+ */
+void xml_string_copy(struct xml_string* string, uint8_t* buffer, size_t length);
+
+int xml_node_name_cmp(struct xml_node* node, const char *name);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+