Add recursive update function (#3069)

* ✨ add recursive update function

doc/examples/update.cpp

@@ -7,12 +7,17 @@ using json = nlohmann::json;
 int main()
 {
     // create two JSON objects
-    json o1 = R"( {"color": "red", "price": 17.99} )"_json;
+    json o1 = R"( {"color": "red", "price": 17.99, "names": {"de": "Flugzeug"}} )"_json;
-    json o2 = R"( {"color": "blue", "speed": 100} )"_json;
+    json o2 = R"( {"color": "blue", "speed": 100, "names": {"en": "plane"}} )"_json;
+    json o3 = o1;
 
-    // add all keys from o2 to o1 (updating "color")
+    // add all keys from o2 to o1 (updating "color", replacing "names")
     o1.update(o2);
 
-    // output updated object o1
+    // add all keys from o2 to o1 (updating "color", merging "names")
+    o3.update(o2, true);
+
+    // output updated object o1 and o3
     std::cout << std::setw(2) << o1 << '\n';
+    std::cout << std::setw(2) << o3 << '\n';
 }

doc/examples/update.output

@@ -1,5 +1,17 @@
 {
   "color": "blue",
+  "names": {
+    "en": "plane"
+  },
+  "price": 17.99,
+  "speed": 100
+}
+{
+  "color": "blue",
+  "names": {
+    "de": "Flugzeug",
+    "en": "plane"
+  },
   "price": 17.99,
   "speed": 100
 }

doc/examples/update__range.cpp

@@ -7,12 +7,17 @@ using json = nlohmann::json;
 int main()
 {
     // create two JSON objects
-    json o1 = R"( {"color": "red", "price": 17.99} )"_json;
+    json o1 = R"( {"color": "red", "price": 17.99, "names": {"de": "Flugzeug"}} )"_json;
-    json o2 = R"( {"color": "blue", "speed": 100} )"_json;
+    json o2 = R"( {"color": "blue", "speed": 100, "names": {"en": "plane"}} )"_json;
+    json o3 = o1;
 
-    // add all keys from o2 to o1 (updating "color")
+    // add all keys from o2 to o1 (updating "color", replacing "names")
     o1.update(o2.begin(), o2.end());
 
-    // output updated object o1
+    // add all keys from o2 to o1 (updating "color", merging "names")
+    o3.update(o2.begin(), o2.end(), true);
+
+    // output updated object o1 and o3
     std::cout << std::setw(2) << o1 << '\n';
+    std::cout << std::setw(2) << o3 << '\n';
 }

doc/examples/update__range.output

@@ -1,5 +1,17 @@
 {
   "color": "blue",
+  "names": {
+    "en": "plane"
+  },
+  "price": 17.99,
+  "speed": 100
+}
+{
+  "color": "blue",
+  "names": {
+    "de": "Flugzeug",
+    "en": "plane"
+  },
   "price": 17.99,
   "speed": 100
 }

doc/mkdocs/docs/api/basic_json/update.md

@@ -2,14 +2,17 @@
 
 ```cpp
 // (1)
-void update(const_reference j);
+void update(const_reference j, bool merge_objects = false);
 
 // (2)
-void update(const_iterator first, const_iterator last);
+void update(const_iterator first, const_iterator last, bool merge_objects = false);
 ```
 
-1. Inserts all values from JSON object `j` and overwrites existing keys.
+1. Inserts all values from JSON object `j`.
-2. Inserts all values from from range `[first, last)` and overwrites existing keys.
+2. Inserts all values from from range `[first, last)`
+
+When `merge_objects` is `#!c false` (default), existing keys are overwritten. When `merge_objects` is `#!c true`,
+recursively merges objects with common keys.
 
 The function is motivated by Python's [dict.update](https://docs.python.org/3.6/library/stdtypes.html#dict.update)
 function.
@@ -19,6 +22,10 @@ function.
 `j` (in)
 :   JSON object to read values from
 
+`merge_objects` (in)
+:   when `#!c true`, existing keys are not overwritten, but contents of objects are merged recursively (default:
+    `#!c false`)
+
 `first` (in)
 :   begin of the range of elements to insert
 
@@ -73,6 +80,63 @@ function.
     --8<-- "examples/update__range.output"
     ```
 
+??? example
+
+    One common usecase for this function is the handling of user settings. Assume your application can configured in
+    some aspects:
+
+    ```json
+    {
+        "color": "red",
+        "active": true,
+        "name": {"de": "Maus", "en": "mouse"}
+    }
+    ```
+
+    The user may override the default settings selectively:
+
+    ```json
+    {
+        "color": "blue",
+        "name": {"es": "ratón"},
+    }
+    ```
+
+    Then `update` manages the merging of default settings and user settings:
+ 
+    ```cpp
+    auto user_settings = json::parse("config.json");
+    auto effective_settings = get_default_settings();
+    effective_settings.update(user_settings);
+    ```
+    
+    Now `effective_settings` contains the default settings, but those keys set by the user are overwritten:
+
+    ```json
+    {
+        "color": "blue",
+        "active": true,
+        "name": {"es": "ratón"}
+    }
+    ```
+
+    Note existing keys were just overwritten. To merge objects, `merge_objects` setting should be set to `#!c true`:
+    
+    ```cpp
+    auto user_settings = json::parse("config.json");
+    auto effective_settings = get_default_settings();
+    effective_settings.update(user_settings, true);
+    ```
+
+    ```json
+    {
+        "color": "blue",
+        "active": true,
+        "name": {"de": "Maus", "en": "mouse", "es": "ratón"}
+    }
+    ```
+
 ## Version history
 
 - Added in version 3.0.0.
+- Added `merge_objects` parameter in 3.10.4.

include/nlohmann/json.hpp

@@ -5954,6 +5954,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     Inserts all values from JSON object @a j and overwrites existing keys.
 
     @param[in] j  JSON object to read values from
+    @param[in] merge_objects  when true, existing keys are not overwritten, but
+                              contents of objects are merged recursively
+                              (default: false)
 
     @throw type_error.312 if called on JSON values other than objects; example:
     `"cannot use update() with string"`
@@ -5965,34 +5968,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
 
     @sa https://docs.python.org/3.6/library/stdtypes.html#dict.update
 
-    @since version 3.0.0
+    @since version 3.0.0, `merge_objects` parameter added in 3.10.4.
     */
-    void update(const_reference j)
+    void update(const_reference j, bool merge_objects = false)
     {
-        // implicitly convert null value to an empty object
+        update(j.begin(), j.end(), merge_objects);
-        if (is_null())
-        {
-            m_type = value_t::object;
-            m_value.object = create<object_t>();
-            assert_invariant();
-        }
-
-        if (JSON_HEDLEY_UNLIKELY(!is_object()))
-        {
-            JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(type_name()), *this));
-        }
-        if (JSON_HEDLEY_UNLIKELY(!j.is_object()))
-        {
-            JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(j.type_name()), *this));
-        }
-
-        for (auto it = j.cbegin(); it != j.cend(); ++it)
-        {
-            m_value.object->operator[](it.key()) = it.value();
-#if JSON_DIAGNOSTICS
-            m_value.object->operator[](it.key()).m_parent = this;
-#endif
-        }
     }
 
     /*!
@@ -6003,12 +5983,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
 
     @param[in] first begin of the range of elements to insert
     @param[in] last end of the range of elements to insert
+    @param[in] merge_objects  when true, existing keys are not overwritten, but
+                              contents of objects are merged recursively
+                              (default: false)
 
     @throw type_error.312 if called on JSON values other than objects; example:
     `"cannot use update() with string"`
-    @throw invalid_iterator.202 if iterator @a first or @a last does does not
+    @throw type_error.312 if iterator @a first or @a last does does not
-    point to an object; example: `"iterators first and last must point to
+    point to an object; example: `"cannot use update() with string"`
-    objects"`
     @throw invalid_iterator.210 if @a first and @a last do not belong to the
     same JSON value; example: `"iterators do not fit"`
 
@@ -6019,9 +6001,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
 
     @sa https://docs.python.org/3.6/library/stdtypes.html#dict.update
 
-    @since version 3.0.0
+    @since version 3.0.0, `merge_objects` parameter added in 3.10.4.
     */
-    void update(const_iterator first, const_iterator last)
+    void update(const_iterator first, const_iterator last, bool merge_objects = false)
     {
         // implicitly convert null value to an empty object
         if (is_null())
@@ -6043,14 +6025,22 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         }
 
         // passed iterators must belong to objects
-        if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_object()
+        if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_object()))
-                                 || !last.m_object->is_object()))
         {
-            JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects", *this));
+            JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(first.m_object->type_name()), *first.m_object));
         }
 
         for (auto it = first; it != last; ++it)
         {
+            if (merge_objects && it.value().is_object())
+            {
+                auto it2 = m_value.object->find(it.key());
+                if (it2 != m_value.object->end())
+                {
+                    it2->second.update(it.value(), true);
+                    continue;
+                }
+            }
             m_value.object->operator[](it.key()) = it.value();
 #if JSON_DIAGNOSTICS
             m_value.object->operator[](it.key()).m_parent = this;

single_include/nlohmann/json.hpp

@@ -23455,6 +23455,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
     Inserts all values from JSON object @a j and overwrites existing keys.
 
     @param[in] j  JSON object to read values from
+    @param[in] merge_objects  when true, existing keys are not overwritten, but
+                              contents of objects are merged recursively
+                              (default: false)
 
     @throw type_error.312 if called on JSON values other than objects; example:
     `"cannot use update() with string"`
@@ -23466,34 +23469,11 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
 
     @sa https://docs.python.org/3.6/library/stdtypes.html#dict.update
 
-    @since version 3.0.0
+    @since version 3.0.0, `merge_objects` parameter added in 3.10.4.
     */
-    void update(const_reference j)
+    void update(const_reference j, bool merge_objects = false)
     {
-        // implicitly convert null value to an empty object
+        update(j.begin(), j.end(), merge_objects);
-        if (is_null())
-        {
-            m_type = value_t::object;
-            m_value.object = create<object_t>();
-            assert_invariant();
-        }
-
-        if (JSON_HEDLEY_UNLIKELY(!is_object()))
-        {
-            JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(type_name()), *this));
-        }
-        if (JSON_HEDLEY_UNLIKELY(!j.is_object()))
-        {
-            JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(j.type_name()), *this));
-        }
-
-        for (auto it = j.cbegin(); it != j.cend(); ++it)
-        {
-            m_value.object->operator[](it.key()) = it.value();
-#if JSON_DIAGNOSTICS
-            m_value.object->operator[](it.key()).m_parent = this;
-#endif
-        }
     }
 
     /*!
@@ -23504,12 +23484,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
 
     @param[in] first begin of the range of elements to insert
     @param[in] last end of the range of elements to insert
+    @param[in] merge_objects  when true, existing keys are not overwritten, but
+                              contents of objects are merged recursively
+                              (default: false)
 
     @throw type_error.312 if called on JSON values other than objects; example:
     `"cannot use update() with string"`
-    @throw invalid_iterator.202 if iterator @a first or @a last does does not
+    @throw type_error.312 if iterator @a first or @a last does does not
-    point to an object; example: `"iterators first and last must point to
+    point to an object; example: `"cannot use update() with string"`
-    objects"`
     @throw invalid_iterator.210 if @a first and @a last do not belong to the
     same JSON value; example: `"iterators do not fit"`
 
@@ -23520,9 +23502,9 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
 
     @sa https://docs.python.org/3.6/library/stdtypes.html#dict.update
 
-    @since version 3.0.0
+    @since version 3.0.0, `merge_objects` parameter added in 3.10.4.
     */
-    void update(const_iterator first, const_iterator last)
+    void update(const_iterator first, const_iterator last, bool merge_objects = false)
     {
         // implicitly convert null value to an empty object
         if (is_null())
@@ -23544,14 +23526,22 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec
         }
 
         // passed iterators must belong to objects
-        if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_object()
+        if (JSON_HEDLEY_UNLIKELY(!first.m_object->is_object()))
-                                 || !last.m_object->is_object()))
         {
-            JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects", *this));
+            JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(first.m_object->type_name()), *first.m_object));
         }
 
         for (auto it = first; it != last; ++it)
         {
+            if (merge_objects && it.value().is_object())
+            {
+                auto it2 = m_value.object->find(it.key());
+                if (it2 != m_value.object->end())
+                {
+                    it2->second.update(it.value(), true);
+                    continue;
+                }
+            }
             m_value.object->operator[](it.key()) = it.value();
 #if JSON_DIAGNOSTICS
             m_value.object->operator[](it.key()).m_parent = this;

test/src/unit-diagnostics.cpp

@@ -97,6 +97,18 @@ TEST_CASE("Better diagnostics")
         CHECK_THROWS_WITH_AS(_ = json::parse(""), "[json.exception.parse_error.101] parse error at line 1, column 1: syntax error while parsing value - unexpected end of input; expected '[', '{', or a literal", json::parse_error);
     }
 
+    SECTION("Wrong type in update()")
+    {
+        json j = {{"foo", "bar"}};
+        json k = {{"bla", 1}};
+
+        CHECK_THROWS_WITH_AS(j.update(k["bla"].begin(), k["bla"].end()), "[json.exception.type_error.312] (/bla) cannot use update() with number", json::type_error);
+        CHECK_THROWS_WITH_AS(j.update(k["bla"]), "[json.exception.type_error.312] (/bla) cannot use update() with number", json::type_error);
+    }
+}
+
+TEST_CASE("Regression tests for extended diagnostics")
+{
     SECTION("Regression test for https://github.com/nlohmann/json/pull/2562#pullrequestreview-574858448")
     {
         CHECK_THROWS_WITH_AS(json({"0", "0"})[1].get<int>(), "[json.exception.type_error.302] (/1) type must be number, but is string", json::type_error);
@@ -110,7 +122,7 @@ TEST_CASE("Better diagnostics")
         CHECK_THROWS_WITH_AS(j.unflatten(), "[json.exception.type_error.315] (/~1foo) values in object must be primitive", json::type_error);
     }
 
-    SECTION("Regression test for https://github.com/nlohmann/json/issues/2838")
+    SECTION("Regression test for issue #2838 - Assertion failure when inserting into arrays with JSON_DIAGNOSTICS set")
     {
         // void push_back(basic_json&& val)
         {
@@ -235,7 +247,7 @@ TEST_CASE("Better diagnostics")
         }
     }
 
-    SECTION("Regression test for https://github.com/nlohmann/json/issues/3032")
+    SECTION("Regression test for issue #3032 - Yet another assertion failure when inserting into arrays with JSON_DIAGNOSTICS set")
     {
         // reference operator[](size_type idx)
         {

test/src/unit-modifiers.cpp

@@ -796,64 +796,89 @@ TEST_CASE("modifiers")
 
     SECTION("update()")
     {
-        json j_object1 = {{"one", "eins"}, {"two", "zwei"}};
+        SECTION("non-recursive (default)")
-        json j_object2 = {{"three", "drei"}, {"two", "zwo"}};
-        json j_array = {1, 2, 3, 4};
-
-        SECTION("const reference")
         {
-            SECTION("proper usage")
+            json j_object1 = {{"one", "eins"}, {"two", "zwei"}};
-            {
+            json j_object2 = {{"three", "drei"}, {"two", "zwo"}};
-                j_object1.update(j_object2);
+            json j_array = {1, 2, 3, 4};
-                CHECK(j_object1 == json({{"one", "eins"}, {"two", "zwo"}, {"three", "drei"}}));
 
-                json j_null;
+            SECTION("const reference")
-                j_null.update(j_object2);
+            {
-                CHECK(j_null == j_object2);
+                SECTION("proper usage")
+                {
+                    j_object1.update(j_object2);
+                    CHECK(j_object1 == json({{"one", "eins"}, {"two", "zwo"}, {"three", "drei"}}));
+
+                    json j_null;
+                    j_null.update(j_object2);
+                    CHECK(j_null == j_object2);
+                }
+
+                SECTION("wrong types")
+                {
+                    CHECK_THROWS_AS(j_array.update(j_object1), json::type_error&);
+                    CHECK_THROWS_WITH(j_array.update(j_object1), "[json.exception.type_error.312] cannot use update() with array");
+
+                    CHECK_THROWS_AS(j_object1.update(j_array), json::type_error&);
+                    CHECK_THROWS_WITH(j_object1.update(j_array), "[json.exception.type_error.312] cannot use update() with array");
+                }
             }
 
-            SECTION("wrong types")
+            SECTION("iterator range")
             {
-                CHECK_THROWS_AS(j_array.update(j_object1), json::type_error&);
+                SECTION("proper usage")
-                CHECK_THROWS_WITH(j_array.update(j_object1), "[json.exception.type_error.312] cannot use update() with array");
+                {
+                    j_object1.update(j_object2.begin(), j_object2.end());
+                    CHECK(j_object1 == json({{"one", "eins"}, {"two", "zwo"}, {"three", "drei"}}));
 
-                CHECK_THROWS_AS(j_object1.update(j_array), json::type_error&);
+                    json j_null;
-                CHECK_THROWS_WITH(j_object1.update(j_array), "[json.exception.type_error.312] cannot use update() with array");
+                    j_null.update(j_object2.begin(), j_object2.end());
+                    CHECK(j_null == j_object2);
+                }
+
+                SECTION("empty range")
+                {
+                    j_object1.update(j_object2.begin(), j_object2.begin());
+                    CHECK(j_object1 == json({{"one", "eins"}, {"two", "zwei"}}));
+                }
+
+                SECTION("invalid iterators")
+                {
+                    json j_other_array2 = {"first", "second"};
+
+                    CHECK_THROWS_AS(j_array.update(j_object2.begin(), j_object2.end()), json::type_error&);
+                    CHECK_THROWS_AS(j_object1.update(j_object1.begin(), j_object2.end()), json::invalid_iterator&);
+                    CHECK_THROWS_AS(j_object1.update(j_array.begin(), j_array.end()), json::type_error&);
+
+                    CHECK_THROWS_WITH(j_array.update(j_object2.begin(), j_object2.end()),
+                                      "[json.exception.type_error.312] cannot use update() with array");
+                    CHECK_THROWS_WITH(j_object1.update(j_object1.begin(), j_object2.end()),
+                                      "[json.exception.invalid_iterator.210] iterators do not fit");
+                    CHECK_THROWS_WITH(j_object1.update(j_array.begin(), j_array.end()),
+                                      "[json.exception.type_error.312] cannot use update() with array");
+                }
             }
         }
 
-        SECTION("iterator range")
+        SECTION("recursive")
         {
-            SECTION("proper usage")
+            SECTION("const reference")
             {
-                j_object1.update(j_object2.begin(), j_object2.end());
+                SECTION("extend object")
-                CHECK(j_object1 == json({{"one", "eins"}, {"two", "zwo"}, {"three", "drei"}}));
+                {
+                    json j1 = {{"string", "s"}, {"numbers", {{"one", 1}}}};
+                    json j2 = {{"string", "t"}, {"numbers", {{"two", 2}}}};
+                    j1.update(j2, true);
+                    CHECK(j1 == json({{"string", "t"}, {"numbers", {{"one", 1}, {"two", 2}}}}));
+                }
 
-                json j_null;
+                SECTION("replace object")
-                j_null.update(j_object2.begin(), j_object2.end());
+                {
-                CHECK(j_null == j_object2);
+                    json j1 = {{"string", "s"}, {"numbers", {{"one", 1}}}};
-            }
+                    json j2 = {{"string", "t"}, {"numbers", 1}};
-
+                    j1.update(j2, true);
-            SECTION("empty range")
+                    CHECK(j1 == json({{"string", "t"}, {"numbers", 1}}));
-            {
+                }
-                j_object1.update(j_object2.begin(), j_object2.begin());
-                CHECK(j_object1 == json({{"one", "eins"}, {"two", "zwei"}}));
-            }
-
-            SECTION("invalid iterators")
-            {
-                json j_other_array2 = {"first", "second"};
-
-                CHECK_THROWS_AS(j_array.update(j_object2.begin(), j_object2.end()), json::type_error&);
-                CHECK_THROWS_AS(j_object1.update(j_object1.begin(), j_object2.end()), json::invalid_iterator&);
-                CHECK_THROWS_AS(j_object1.update(j_array.begin(), j_array.end()), json::invalid_iterator&);
-
-                CHECK_THROWS_WITH(j_array.update(j_object2.begin(), j_object2.end()),
-                                  "[json.exception.type_error.312] cannot use update() with array");
-                CHECK_THROWS_WITH(j_object1.update(j_object1.begin(), j_object2.end()),
-                                  "[json.exception.invalid_iterator.210] iterators do not fit");
-                CHECK_THROWS_WITH(j_object1.update(j_array.begin(), j_array.end()),
-                                  "[json.exception.invalid_iterator.202] iterators first and last must point to objects");
             }
         }
     }