mariadb/sql/my_json_writer.cc

/* Copyright (C) 2014, 2021, MariaDB Corporation.

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; version 2 of the License.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA */

#include "my_global.h"
#include <json_lib.h>
#include "my_json_writer.h"

#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)

bool Json_writer::named_item_expected() const
{
  return named_items_expectation.size()
      && named_items_expectation.back();
}
#endif

void Json_writer::append_indent()
{
  if (!document_start)
    output.append('\n');
  for (int i=0; i< indent_level; i++)
    output.append(' ');
}

inline void Json_writer::on_start_object()
{
#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
  if(!fmt_helper.is_making_writer_calls())
  {
    if (got_name != named_item_expected())
    {
      sql_print_error(got_name
                      ? "Json_writer got a member name which is not expected.\n"
                      : "Json_writer: a member name was expected.\n");
      VALIDITY_ASSERT(got_name == named_item_expected());
    }
    named_items_expectation.push_back(true);
  }
#endif
  fmt_helper.on_start_object();
}

void Json_writer::start_object()
{
  on_start_object();

  if (!element_started)
    start_element();

  output.append('{');
  indent_level+=INDENT_SIZE;
  first_child=true;
  element_started= false;
  document_start= false;
#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
  got_name= false;
  named_items.emplace();
#endif
}

void Json_writer::start_array()
{
#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
  if(!fmt_helper.is_making_writer_calls())
  {
    VALIDITY_ASSERT(got_name == named_item_expected());
    named_items_expectation.push_back(false);
    got_name= false;
    if (document_start)
      named_items.emplace();
  }
#endif

  if (fmt_helper.on_start_array())
    return;

  if (!element_started)
    start_element();

  output.append('[');
  indent_level+=INDENT_SIZE;
  first_child=true;
  element_started= false;
  document_start= false;
}


void Json_writer::end_object()
{
#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
  VALIDITY_ASSERT(named_item_expected());
  named_items_expectation.pop_back();
  VALIDITY_ASSERT(!got_name);
  got_name= false;
  VALIDITY_ASSERT(named_items.size());
  named_items.pop();
#endif
  indent_level-=INDENT_SIZE;
  if (!first_child)
    append_indent();
  first_child= false;
  output.append('}');
}


void Json_writer::end_array()
{
#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
  VALIDITY_ASSERT(!named_item_expected());
  named_items_expectation.pop_back();
  got_name= false;
#endif
  if (fmt_helper.on_end_array())
    return;
  indent_level-=INDENT_SIZE;
  if (!first_child)
    append_indent();
  output.append(']');
}


Json_writer& Json_writer::add_member(const char *name)
{
  size_t len= strlen(name);
  return add_member(name, len);
}

Json_writer& Json_writer::add_member(const char *name, size_t len)
{
  if (!fmt_helper.on_add_member(name, len))
  {
    // assert that we are in an object
    DBUG_ASSERT(!element_started);
    start_element();

    output.append('"');
    output.append(name, len);
    output.append(STRING_WITH_LEN("\": "));
  }
#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
  if (!fmt_helper.is_making_writer_calls())
  {
    VALIDITY_ASSERT(!got_name);
    got_name= true;
    VALIDITY_ASSERT(named_items.size());
    auto& named_items_keys= named_items.top();
    auto emplaced= named_items_keys.emplace(name, len);
    auto is_uniq_key= emplaced.second;
    if(!is_uniq_key)
    {
      sql_print_error("Duplicated key: %s\n", emplaced.first->c_str());
      VALIDITY_ASSERT(is_uniq_key);
    }
  }
#endif
  return *this;
}


/* 
  Used by formatting helper to print something that is formatted by the helper.
  We should only separate it from the previous element.
*/

void Json_writer::start_sub_element()
{
  //element_started= true;
  if (first_child)
    first_child= false;
  else
    output.append(',');

  append_indent();
}


void Json_writer::start_element()
{
  element_started= true;

  if (first_child)
    first_child= false;
  else
    output.append(',');

  append_indent();
}

void Json_writer::add_ll(longlong val)
{
  char buf[64];
  size_t len= my_snprintf(buf, sizeof(buf), "%lld", val);
  add_escaped_quoted_str(buf, len);
}

void Json_writer::add_ull(ulonglong val)
{
  char buf[64];
  size_t len= my_snprintf(buf, sizeof(buf), "%llu", val);
  add_escaped_quoted_str(buf, len);
}


/* Add a memory size, printing in KiB, MiB if necessary */
void Json_writer::add_size(longlong val)
{
  char buf[64];
  size_t len;
  if (val < 1024) 
    len= my_snprintf(buf, sizeof(buf), "%lld", val);
  else if (val < 1024*1024*16)
    /* Values less than 16MB are specified in KiB for precision */
    len= my_snprintf(buf, sizeof(buf), "%lldKiB", val/1024);
  else
    len= my_snprintf(buf, sizeof(buf), "%lldMiB", val/(1024*1024));
  add_str(buf, len);
}


void Json_writer::add_double(double val)
{
  char buf[64];
  size_t len= my_snprintf(buf, sizeof(buf), "%-.11lg", val);
  add_escaped_quoted_str(buf, len);
}


void Json_writer::add_bool(bool val)
{
  if (val)
    add_escaped_quoted_str("true", (size_t) 4);
  else
    add_escaped_quoted_str("false", (size_t) 5);
}


void Json_writer::add_null()
{
  add_escaped_quoted_str("null", (size_t) 4);
}


inline bool Json_writer::on_add_str(const char *str, size_t num_bytes)
{
#if !defined(NDEBUG) || defined(JSON_WRITER_UNIT_TEST)
  got_name= false;
#endif
  bool helped= fmt_helper.on_add_str(str, num_bytes);
  return helped;
}

void Json_writer::add_str(const char *str)
{
  size_t len= strlen(str);
  add_str(str, len);
}

/*
  @brief
    Add a string pointed by str of num_bytes length, escaping it if needed.

  @param
    str         String in utf8mb4.
    num_bytes   Length of string in bytes (not characters).
*/

void Json_writer::add_str(const char* str, size_t num_bytes)
{
  int rc;
  /* Initial buffer, json_escape_to_string will alloc larger if needed */
  StringBuffer<128> buf;
  if ((rc= json_escape_to_string(str, num_bytes, &my_charset_utf8mb4_bin, &buf)))
  {
    if (rc == JSON_ERROR_ILLEGAL_SYMBOL)
      str= "\"String with illegal unicode symbol\"";
    else
    {
      DBUG_ASSERT(rc == JSON_ERROR_OUT_OF_SPACE);
      str= "\"Out of memory writing JSON\"";
    }
    num_bytes= strlen(str);
  }
  else
  {
    str= buf.ptr();
    num_bytes= buf.length();
  }
  add_escaped_quoted_str(str, num_bytes);
}


/*
  @brief
    Add a value that the caller has already escaped and quoted as needed
    for JSON (e.g. a quoted string, or a bare number/bool/null literal).
*/

void Json_writer::add_escaped_quoted_str(const char* str, size_t num_bytes)
{
  VALIDITY_ASSERT(fmt_helper.is_making_writer_calls() ||
                  got_name == named_item_expected());
  DBUG_ASSERT((str[0] == '"' && str[num_bytes - 1] == '"') ||  // string
              (str[0] >= '0' && str[0] <= '9') || str[0] == '-' ||  // number
              str[0] == 't' || str[0] == 'f' || str[0] == 'n');  // true/false/null
  if (on_add_str(str, num_bytes))
    return;

  if (!element_started)
    start_element();

  output.append(str, num_bytes);
  element_started= false;
}

void Json_writer::add_str(const String &str)
{
  uint32 offs;
  /* str may be in any charset. Convert to utf8mb4 if necessary */
  if (String::needs_conversion(str.length(), str.charset(),
                               &my_charset_utf8mb4_bin, &offs))
  {
    uint err;
    StringBuffer<128> converted_str;
    if (converted_str.copy(&str, &my_charset_utf8mb4_bin, &err))
    {
      const char *errstr= "Conversion error writing JSON";
      converted_str.set(errstr, strlen(errstr), &my_charset_utf8mb4_bin);
    }
    add_str(converted_str.ptr(), converted_str.length());
  }
  else
    add_str(str.ptr(), str.length());
}

#ifdef ENABLED_JSON_WRITER_CONSISTENCY_CHECKS
thread_local std::vector<bool> Json_writer_struct::named_items_expectation;
#endif

Json_writer_temp_disable::Json_writer_temp_disable(THD *thd_arg)
{
  thd= thd_arg;
  thd->opt_trace.disable_tracing_if_required();
}
Json_writer_temp_disable::~Json_writer_temp_disable()
{
  thd->opt_trace.enable_tracing_if_required();
}

bool Single_line_formatting_helper::on_add_member(const char *name,
                                                  size_t len)
{
  DBUG_ASSERT(state== INACTIVE || state == DISABLED);
  if (memchr(name, 0, len))
    return false;
  if (state != DISABLED)
  {
    // remove everything from the array
    buf_ptr= buffer;

    //append member name to the array
    if (len < MAX_LINE_LEN)
    {
      memcpy(buf_ptr, name, len);
      buf_ptr+=len;
      *(buf_ptr++)= 0;

      line_len= owner->indent_level + (uint)len + 1;
      state= ADD_MEMBER;
      return true; // handled
    }
  }
  return false; // not handled
}


bool Single_line_formatting_helper::on_start_array()
{
  if (state == ADD_MEMBER)
  {
    state= IN_ARRAY;
    return true; // handled
  }
  else
  {
    if (state != DISABLED)
      state= INACTIVE;
    // TODO: what if we have accumulated some stuff already? shouldn't we
    // flush it?
    return false; // not handled
  }
}


bool Single_line_formatting_helper::on_end_array()
{
  if (state == IN_ARRAY)
  {
    flush_on_one_line();
    state= INACTIVE;
    return true; // handled
  }
  return false; // not handled
}


void Single_line_formatting_helper::on_start_object()
{
  // Nested objects will not be printed on one line
  disable_and_flush();
}


bool Single_line_formatting_helper::on_add_str(const char *str,
                                               size_t len)
{
  if (state == IN_ARRAY)
  {
    if (memchr(str, 0, len))
    {
      disable_and_flush();
      return false;
    }

    // New length will be:
    //  str,
    // str already includes any quotes it needs; comma + space = 2
    if (line_len + len + 2 > MAX_LINE_LEN)
    {
      disable_and_flush();
      return false; // didn't handle the last element
    }

    //append string to array
    memcpy(buf_ptr, str, len);
    buf_ptr+=len;
    *(buf_ptr++)= 0;
    line_len += (uint)len + 2;
    return true; // handled
  }

  disable_and_flush();
  return false; // not handled
}


/*
  Append everything accumulated to the output on one line
*/

void Single_line_formatting_helper::flush_on_one_line()
{
  owner->start_sub_element();
  char *ptr= buffer;
  int nr= 0;
  while (ptr < buf_ptr)
  {
    char *str= ptr;

    if (nr == 0)
    {
      owner->output.append('"');
      owner->output.append(str);
      owner->output.append(STRING_WITH_LEN("\": "));
      owner->output.append('[');
    }
    else
    {
      if (nr != 1)
        owner->output.append(STRING_WITH_LEN(", "));
      owner->output.append(str);
    }
    nr++;

    while (*ptr!=0)
      ptr++;
    ptr++;
  }
  owner->output.append(']');
  /* We've printed out the contents of the buffer, mark it as empty */
  buf_ptr= buffer;
}


void Single_line_formatting_helper::disable_and_flush()
{
  if (state == DISABLED)
    return;

  bool start_array= (state == IN_ARRAY);
  state= DISABLED;
  // deactivate ourselves and flush all accumulated calls.
  char *ptr= buffer;
  int nr= 0;
  while (ptr < buf_ptr)
  {
    char *str= ptr;
    size_t len= strlen(str);

    if (nr == 0)
    {
      owner->add_member(str, len);
      if (start_array)
        owner->start_array();
    }
    else
    {
      //if (nr == 1)
      //  owner->start_array();
      owner->add_escaped_quoted_str(str, len);
    }
    nr++;
    ptr+= len+1;
  }
  buf_ptr= buffer;
  state= INACTIVE;
}

/*
  @brief
    Escape and quote a JSON string and save it into *out.

  @detail
    There's no way to tell how much space is needed for the output.
    Start with a small string and increase its size until json_escape()
    succeeds.
*/

int json_escape_to_string(const char *str, size_t len, CHARSET_INFO *cs,
                          String *out)
{
  /* Safety: assert that str doesn't point into *out. */
  DBUG_ASSERT(!out->alloced_length() ||
              str < out->ptr() || str >= out->ptr() + out->alloced_length());

  // Make sure 'out' has some memory allocated.
  if (!out->alloced_length() && out->alloc(128))
    return JSON_ERROR_OUT_OF_SPACE;

  while (1)
  {
    uchar *buf= (uchar*)out->ptr();
    out->length(out->alloced_length());
    const uchar *str_ptr= (const uchar*)str;

    // first, and the last bytes of buf are used for double quote
    int res= json_escape(cs,
                         str_ptr,
                         str_ptr + len,
                         &my_charset_utf8mb4_bin,
                         buf+1, buf + out->length()-1);
    if (res >= 0)
    {
      buf[0]= '"';
      buf[res+1]= '"';
      out->length(res+2);
      return 0; // Ok
    }

    if (res != JSON_ERROR_OUT_OF_SPACE)
      return res; // Some conversion error

    // Out of space error. Try with a bigger buffer
    if (out->alloc(out->alloced_length()*2))
      return JSON_ERROR_OUT_OF_SPACE;
  }
}

int json_escape_to_string(const String *str, String *out)
{
  return json_escape_to_string(str->ptr(), str->length(), str->charset(), out);
}