Make SAX output locale-independent (#4505)

* 🐛 make SAX output locale-independent #4084

* ✅ add test

* ✅ add test

* ✅ add test

* ✅ add test

* ✅ add test

* ✅ add test

* 🐛 make SAX output locale-independent #4084

* 🐛 make SAX output locale-independent #4084

* 🐛 make SAX output locale-independent #4084

* 🐛 make SAX output locale-independent #4084

* 🐛 make SAX output locale-independent #4084

* 🐛 make SAX output locale-independent #4084

* 🐛 make SAX output locale-independent #4084

* 🐛 make SAX output locale-independent #4084

* 🐛 make SAX output locale-independent #4084

* 🐛 make SAX output locale-independent #4084

* 🐛 make SAX output locale-independent #4084

* 🐛 make SAX output locale-independent #4084

.github/workflows/ubuntu.yml

@@ -104,23 +104,29 @@ jobs:
 
   ci_test_coverage:
     runs-on: ubuntu-latest
-    container: ghcr.io/nlohmann/json-ci:v2.4.0
     permissions:
       contents: read
       checks: write
     steps:
       - uses: actions/checkout@v4
+      - name: Install dependencies and de_DE locale
+        run: |
+          sudo apt-get clean
+          sudo apt-get update
+          sudo apt-get install -y build-essential cmake lcov ninja-build make locales gcc-multilib g++-multilib
+          sudo locale-gen de_DE
+          sudo update-locale
       - name: Run CMake
         run: cmake -S . -B build -DJSON_CI=On
       - name: Build
         run: cmake --build build --target ci_test_coverage
       - name: Archive coverage report
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: code-coverage-report
           path: ${{ github.workspace }}/build/html
       - name: Publish report to Coveralls
-        uses: coverallsapp/github-action@master
+        uses: coverallsapp/github-action@v2.3.4
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           path-to-lcov: ${{ github.workspace }}/build/json.info.filtered.noexcept

include/nlohmann/detail/input/lexer.hpp

@@ -1049,6 +1049,7 @@ scan_number_zero:
             case '.':
             {
                 add(decimal_point_char);
+                decimal_point_position = token_buffer.size() - 1;
                 goto scan_number_decimal1;
             }
 
@@ -1085,6 +1086,7 @@ scan_number_any1:
             case '.':
             {
                 add(decimal_point_char);
+                decimal_point_position = token_buffer.size() - 1;
                 goto scan_number_decimal1;
             }
 
@@ -1322,6 +1324,7 @@ scan_number_done:
     {
         token_buffer.clear();
         token_string.clear();
+        decimal_point_position = std::string::npos;
         token_string.push_back(char_traits<char_type>::to_char_type(current));
     }
 
@@ -1430,6 +1433,11 @@ scan_number_done:
     /// return current string value (implicitly resets the token; useful only once)
     string_t& get_string()
     {
+        // translate decimal points from locale back to '.' (#4084)
+        if (decimal_point_char != '.' && decimal_point_position != std::string::npos)
+        {
+            token_buffer[decimal_point_position] = '.';
+        }
         return token_buffer;
     }
 
@@ -1627,6 +1635,8 @@ scan_number_done:
 
     /// the decimal point
     const char_int_type decimal_point_char = '.';
+    /// the position of the decimal point in the input
+    std::size_t decimal_point_position = std::string::npos;
 };
 
 }  // namespace detail

single_include/nlohmann/json.hpp

@@ -8518,6 +8518,7 @@ scan_number_zero:
             case '.':
             {
                 add(decimal_point_char);
+                decimal_point_position = token_buffer.size() - 1;
                 goto scan_number_decimal1;
             }
 
@@ -8554,6 +8555,7 @@ scan_number_any1:
             case '.':
             {
                 add(decimal_point_char);
+                decimal_point_position = token_buffer.size() - 1;
                 goto scan_number_decimal1;
             }
 
@@ -8791,6 +8793,7 @@ scan_number_done:
     {
         token_buffer.clear();
         token_string.clear();
+        decimal_point_position = std::string::npos;
         token_string.push_back(char_traits<char_type>::to_char_type(current));
     }
 
@@ -8899,6 +8902,11 @@ scan_number_done:
     /// return current string value (implicitly resets the token; useful only once)
     string_t& get_string()
     {
+        // translate decimal points from locale back to '.' (#4084)
+        if (decimal_point_char != '.' && decimal_point_position != std::string::npos)
+        {
+            token_buffer[decimal_point_position] = '.';
+        }
         return token_buffer;
     }
 
@@ -9096,6 +9104,8 @@ scan_number_done:
 
     /// the decimal point
     const char_int_type decimal_point_char = '.';
+    /// the position of the decimal point in the input
+    std::size_t decimal_point_position = std::string::npos;
 };
 
 }  // namespace detail

tests/src/unit-locale-cpp.cpp

@@ -0,0 +1,161 @@
+//     __ _____ _____ _____
+//  __|  |   __|     |   | |  JSON for Modern C++ (supporting code)
+// |  |  |__   |  |  | | | |  version 3.11.3
+// |_____|_____|_____|_|___|  https://github.com/nlohmann/json
+//
+// SPDX-FileCopyrightText: 2013-2023 Niels Lohmann <https://nlohmann.me>
+// SPDX-License-Identifier: MIT
+
+#include "doctest_compatibility.h"
+
+#define JSON_TESTS_PRIVATE
+#include <nlohmann/json.hpp>
+using nlohmann::json;
+
+#include <clocale>
+
+struct ParserImpl final: public nlohmann::json_sax<json>
+{
+    bool null() override
+    {
+        return true;
+    }
+    bool boolean(bool /*val*/) override
+    {
+        return true;
+    }
+    bool number_integer(json::number_integer_t /*val*/) override
+    {
+        return true;
+    }
+    bool number_unsigned(json::number_unsigned_t /*val*/) override
+    {
+        return true;
+    }
+    bool number_float(json::number_float_t /*val*/, const json::string_t& s) override
+    {
+        float_string_copy = s;
+        return true;
+    }
+    bool string(json::string_t& /*val*/) override
+    {
+        return true;
+    }
+    bool binary(json::binary_t& /*val*/) override
+    {
+        return true;
+    }
+    bool start_object(std::size_t /*val*/) override
+    {
+        return true;
+    }
+    bool key(json::string_t& /*val*/) override
+    {
+        return true;
+    }
+    bool end_object() override
+    {
+        return true;
+    }
+    bool start_array(std::size_t /*val*/) override
+    {
+        return true;
+    }
+    bool end_array() override
+    {
+        return true;
+    }
+    bool parse_error(std::size_t /*val*/, const std::string& /*val*/, const nlohmann::detail::exception& /*val*/) override
+    {
+        return false;
+    }
+
+    ~ParserImpl() override;
+
+    ParserImpl()
+        : float_string_copy("not set")
+    {}
+
+    ParserImpl(const ParserImpl& other)
+        : float_string_copy(other.float_string_copy)
+    {}
+
+    ParserImpl(ParserImpl&& other) noexcept
+        : float_string_copy(std::move(other.float_string_copy))
+    {}
+
+    ParserImpl& operator=(const ParserImpl& other)
+    {
+        if (this != &other)
+        {
+            float_string_copy = other.float_string_copy;
+        }
+        return *this;
+    }
+
+    ParserImpl& operator=(ParserImpl&& other) noexcept
+    {
+        if (this != &other)
+        {
+            float_string_copy = std::move(other.float_string_copy);
+        }
+        return *this;
+    }
+
+    json::string_t float_string_copy;
+};
+
+ParserImpl::~ParserImpl() = default;
+
+TEST_CASE("locale-dependent test (LC_NUMERIC=C)")
+{
+    WARN_MESSAGE(std::setlocale(LC_NUMERIC, "C") != nullptr, "could not set locale");
+
+    SECTION("check if locale is properly set")
+    {
+        std::array<char, 6> buffer = {};
+        CHECK(std::snprintf(buffer.data(), buffer.size(), "%.2f", 12.34) == 5); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg)
+        CHECK(std::string(buffer.data()) == "12.34");
+    }
+
+    SECTION("parsing")
+    {
+        CHECK(json::parse("12.34").dump() == "12.34");
+    }
+
+    SECTION("SAX parsing")
+    {
+        ParserImpl sax {};
+        json::sax_parse( "12.34", &sax );
+        CHECK(sax.float_string_copy == "12.34");
+    }
+}
+
+TEST_CASE("locale-dependent test (LC_NUMERIC=de_DE)")
+{
+    if (std::setlocale(LC_NUMERIC, "de_DE") != nullptr)
+    {
+        SECTION("check if locale is properly set")
+        {
+            std::array<char, 6> buffer = {};
+            CHECK(std::snprintf(buffer.data(), buffer.size(), "%.2f", 12.34) == 5); // NOLINT(cppcoreguidelines-pro-type-vararg,hicpp-vararg)
+            CHECK(std::string(buffer.data()) == "12,34");
+        }
+
+        SECTION("parsing")
+        {
+            CHECK(json::parse("12.34").dump() == "12.34");
+        }
+
+        SECTION("SAX parsing")
+        {
+            ParserImpl sax{};
+            json::sax_parse("12.34", &sax);
+            CHECK(sax.float_string_copy == "12.34");
+        }
+    }
+    else
+    {
+        MESSAGE("locale de_DE is not usable");
+    }
+}