Add fuzz testing for the JSON, ini, and pack readers.

Add libFuzzer targets for the JSON, ini, and pack readers in test/fuzz. Each target runs an input through the reader and discards the errors thrown for invalid input, while a failed assertion or a sanitizer report aborts. The targets are linked with the objects of the pgbackrest binary when the fuzz-engine meson option is set.

test/fuzz/build.sh builds the targets and seed corpora in the environment that OSS-Fuzz defines, so an OSS-Fuzz project can call it if pgBackRest is accepted. Until then the fuzz workflow builds the targets with clang, AddressSanitizer, and UndefinedBehaviorSanitizer and runs each one for a fixed number of runs. The seed is derived from the commit hash and ASLR and corpus reloading are disabled, so a failed run can be reproduced.

Fuzzing found or led to the bugs fixed in these commits:

81704daaf Check the mem context stack bounds in production builds.
4b8e3f438 Limit the depth of nested containers when reading JSON.
b623071a2 Fix reads past the end of JSON strings ending in an escape.
2b919843a Reject unexpected commas when reading JSON.
1808295cc Reject \u0000 when reading JSON strings.
b63357ee8 Accept hex digits in JSON \u escapes when skipping.
6963016c7 Error on invalid field ids when reading packs.
0966ebbb3 Check pack field sizes against the bytes remaining in a buffer.
950c1e149 Error on zero-size pack fields.

.github/workflows/test-fuzz.yml

@@ -0,0 +1,77 @@
+# Build the fuzz targets in test/fuzz with clang and libFuzzer, instrumented for AddressSanitizer and UndefinedBehaviorSanitizer,
+# and run each one from its seed corpus for a fixed number of runs. A crash, a sanitizer report, or a failed assert fails the job
+# and the inputs that caused them are uploaded.
+name: fuzz
+
+on: workflow_call
+
+# Limit permissions on the repository
+permissions:
+  contents: read
+
+jobs:
+  fuzz:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v6
+        with:
+          path: pgbackrest
+
+      - name: Update Apt
+        uses: ./pgbackrest/.github/action/apt
+
+      - name: Install
+        run: |
+          DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive sudo apt-get install -y clang libclang-rt-dev zip unzip \
+            pkg-config meson ninja-build python3-yaml zlib1g-dev libssl-dev libxml2-dev libpq-dev libbz2-dev liblz4-dev libzstd-dev
+
+      # The flags are the ones OSS-Fuzz uses for each sanitizer, combined into a single build
+      - name: Build
+        env:
+          CC: clang
+          CFLAGS: >-
+            -O1 -fno-omit-frame-pointer -gline-tables-only -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address,undefined
+            -fno-sanitize-recover=undefined -fsanitize=fuzzer-no-link
+          LIB_FUZZING_ENGINE: -fsanitize=fuzzer
+        run: |
+          mkdir -p fuzz/out fuzz/work
+          OUT=${GITHUB_WORKSPACE?}/fuzz/out WORK=${GITHUB_WORKSPACE?}/fuzz/work ${GITHUB_WORKSPACE?}/pgbackrest/test/fuzz/build.sh
+
+      # Run each target with a seed from the commit hash and a fixed number of runs, with ASLR and corpus reloading disabled. The
+      # same commit generates the same inputs. Every target runs even when one fails.
+      - name: Run
+        run: |
+          cd fuzz
+          mkdir crash
+          seed=$((16#${GITHUB_SHA:0:7} + 1))
+          result=0
+
+          # Target and number of runs, about a minute each
+          for fuzz in ini:100000 json:200000 pack:500000; do
+              name=fuzz-${fuzz%%:*}
+              runs=${fuzz##*:}
+              option="-seed=${seed} -runs=${runs} -reload=0 -timeout=25 -rss_limit_mb=2560"
+
+              mkdir -p corpus/${name}
+
+              if test -f out/${name}_seed_corpus.zip; then
+                  unzip -q out/${name}_seed_corpus.zip -d corpus/${name}
+              fi
+
+              if ! setarch -R out/${name} ${option} -artifact_prefix=crash/${name}- corpus/${name}; then
+                  echo "reproduce from the seed corpus with: setarch -R ${name} ${option}"
+                  result=1
+              fi
+          done
+
+          exit ${result}
+
+      - name: Upload Crash
+        if: failure()
+        uses: actions/upload-artifact@v7
+        with:
+          name: fuzz-crash
+          path: fuzz/crash
+          if-no-files-found: ignore

.github/workflows/test.yml

@@ -60,3 +60,6 @@ jobs:
 
   misc:
     uses: ./.github/workflows/test-misc.yml
+
+  fuzz:
+    uses: ./.github/workflows/test-fuzz.yml

meson_options.txt

@@ -1,5 +1,6 @@
 option('configdir', type: 'string', value: '/etc/pgbackrest', description: 'Configuration directory')
 option('fatal-errors', type: 'boolean', value: false, description: 'Stop compilation on first error')
+option('fuzz-engine', type: 'string', value: '', description: 'Link args of the fuzzing engine for fuzz targets (empty to disable)')
 option('libssh2', type: 'feature', value: 'auto', description: 'Enable SFTP storage support')
 option('libzstd', type: 'feature', value: 'auto', description: 'Enable Zstandard compression support')
 option('libsystemd', type: 'feature', value: 'auto', description: 'Enable systemd notify support')

test/fuzz/build.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+####################################################################################################################################
+# Build the fuzz targets in test/fuzz into $OUT, each with a seed corpus from test/fuzz/corpus when one exists. The environment is
+# the one OSS-Fuzz defines: CC and CFLAGS instrument the build for the sanitizers and the fuzzing engine, LIB_FUZZING_ENGINE is
+# linked into each target, and WORK is where the build goes. Debug enables asserts, which abort the target when they fail.
+####################################################################################################################################
+set -eu
+
+repo=$(cd "$(dirname "$0")/../.." && pwd)
+
+meson setup "$WORK/build" "$repo" --buildtype=plain -Ddebug=true -Db_ndebug=false -Dfuzz-engine="$LIB_FUZZING_ENGINE"
+meson compile -C "$WORK/build"
+
+for fuzz in "$WORK"/build/test/fuzz-*; do
+    # Skip the object directories
+    [ -f "$fuzz" ] || continue
+
+    name=$(basename "$fuzz")
+    corpus="$repo/test/fuzz/corpus/${name#fuzz-}"
+
+    cp "$fuzz" "$OUT/"
+
+    if [ -d "$corpus" ]; then
+        zip -jq "$OUT/${name}_seed_corpus.zip" "$corpus"/*
+    fi
+done

test/fuzz/corpus/ini/config.ini

@@ -0,0 +1,14 @@
+# Comment
+[global]
+repo1-path=/var/lib/pgbackrest
+repo1-retention-full = 2
+  log-level-console=info
+
+[global:archive-push]
+compress-level=3
+
+[demo]
+pg1-path=/var/lib/postgresql/17/main
+db-include=db1
+db-include=db2
+recovery-option=primary_conninfo=host=pg1 user=replicator

test/fuzz/corpus/ini/info.ini

@@ -0,0 +1,17 @@
+[backrest]
+backrest-checksum="b34b238ce89d8e1365c9e392ce59e7b03342ceb9"
+backrest-format=5
+backrest-version="2.60"
+
+[db]
+db-catalog-version=202307071
+db-control-version=1300
+db-id=1
+db-system-id=7302385934293858112
+db-version="17"
+
+[db:history]
+1={"db-catalog-version":202307071,"db-control-version":1300,"db-system-id":7302385934293858112,"db-version":"17"}
+
+[target:file]
+pg_data/key=with=equals={"checksum":"06d06bb31b570b94d7b4325f511f853dbe771c21","size":8192,"timestamp":1565282114}

test/fuzz/corpus/json/object.json

@@ -0,0 +1 @@
+{"backup-archive-start":"000000010000000000000002","backup-info-size":20162900,"backup-lsn-start":"0/2000028","backup-prior":null,"backup-reference":["20250101-000000F","20250102-000000D"],"option-archive-check":true,"option-compress":false,"db-id":1,"time":-1}

test/fuzz/corpus/json/value.json

@@ -0,0 +1 @@
+[" \"quote\" \\ \/ \b\f\n\r\t \u00e9 \u0041", 18446744073709551615, -9223372036854775808, 0, true, false, null, [], {}, [[[{"a":[{}]}]]]]

test/fuzz/fuzz.c

@@ -0,0 +1,69 @@
+/***********************************************************************************************************************************
+Fuzz Target Driver
+***********************************************************************************************************************************/
+#include <build.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "common/debug.h"
+#include "common/error/error.h"
+#include "common/macro.h"
+#include "common/memContext.h"
+#include "common/stackTrace.h"
+
+#include "fuzz.h"
+
+/***********************************************************************************************************************************
+Fuzzing engine entry points
+***********************************************************************************************************************************/
+int LLVMFuzzerInitialize(int *argc, char ***argv);
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
+
+/**********************************************************************************************************************************/
+int
+LLVMFuzzerInitialize(int *const argc, char ***const argv)
+{
+    FUNCTION_TEST_BEGIN();
+        (void)argc;                                                 // Command line arguments are not used
+        (void)argv;
+    FUNCTION_TEST_END();
+
+    // Set stack trace and mem context error cleanup handlers
+    static const ErrorHandlerFunction errorHandlerList[] = {stackTraceClean, memContextClean};
+    errorHandlerSet(errorHandlerList, LENGTH_OF(errorHandlerList));
+
+    FUNCTION_TEST_RETURN(INT, 0);
+}
+
+/**********************************************************************************************************************************/
+int
+LLVMFuzzerTestOneInput(const uint8_t *const data, const size_t size)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM_P(VOID, data);
+        FUNCTION_TEST_PARAM(SIZE, size);
+    FUNCTION_TEST_END();
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        TRY_BEGIN()
+        {
+            fuzzTarget(BUF(data, size));
+        }
+        CATCH_ANY()
+        {
+            // Errors thrown for invalid input are expected
+        }
+        CATCH_FATAL()
+        {
+            fprintf(stderr, "%s: %s\n%s\n", errorName(), errorMessage(), errorStackTrace());
+            fflush(stderr);
+            abort();
+        }
+        TRY_END();
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    FUNCTION_TEST_RETURN(INT, 0);
+}

test/fuzz/fuzz.h

@@ -0,0 +1,16 @@
+/***********************************************************************************************************************************
+Fuzz Target Driver
+***********************************************************************************************************************************/
+#ifndef TEST_FUZZ_FUZZ_H
+#define TEST_FUZZ_FUZZ_H
+
+#include "common/type/buffer.h"
+
+/***********************************************************************************************************************************
+Functions
+***********************************************************************************************************************************/
+// Process an input from the fuzzing engine. Each target defines this function. Errors thrown for invalid input are discarded but a
+// fatal error aborts.
+void fuzzTarget(const Buffer *input);
+
+#endif

test/fuzz/ini.c

@@ -0,0 +1,52 @@
+/***********************************************************************************************************************************
+Fuzz Ini
+***********************************************************************************************************************************/
+#include <build.h>
+
+#include "common/debug.h"
+#include "common/ini.h"
+#include "common/io/bufferRead.h"
+
+#include "fuzz.h"
+
+/**********************************************************************************************************************************/
+void
+fuzzTarget(const Buffer *const input)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(BUFFER, input);
+    FUNCTION_TEST_END();
+
+    // Read strict ini with JSON values, e.g. info files and manifests
+    TRY_BEGIN()
+    {
+        iniValid(iniNewP(ioBufferReadNew(input), .strict = true));
+    }
+    CATCH_ANY()
+    {
+        // Errors thrown for invalid input are expected
+    }
+    TRY_END();
+
+    // Read ini into a store and get every value, e.g. config files
+    const Ini *const ini = iniNewP(ioBufferReadNew(input), .store = true);
+    const StringList *const sectionList = iniSectionList(ini);
+
+    for (unsigned int sectionIdx = 0; sectionIdx < strLstSize(sectionList); sectionIdx++)
+    {
+        const String *const section = strLstGet(sectionList, sectionIdx);
+        const StringList *const keyList = iniSectionKeyList(ini, section);
+
+        for (unsigned int keyIdx = 0; keyIdx < strLstSize(keyList); keyIdx++)
+        {
+            const String *const key = strLstGet(keyList, keyIdx);
+
+            if (iniSectionKeyIsList(ini, section, key))
+                iniGetList(ini, section, key);
+            else
+                iniGet(ini, section, key);
+        }
+    }
+
+    FUNCTION_TEST_RETURN_VOID();
+}

test/fuzz/json.c

@@ -0,0 +1,36 @@
+/***********************************************************************************************************************************
+Fuzz JSON
+***********************************************************************************************************************************/
+#include <build.h>
+
+#include "common/debug.h"
+#include "common/type/json.h"
+
+#include "fuzz.h"
+
+/**********************************************************************************************************************************/
+void
+fuzzTarget(const Buffer *const input)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(BUFFER, input);
+    FUNCTION_TEST_END();
+
+    const String *const json = strNewBuf(input);
+
+    // Skip the JSON
+    TRY_BEGIN()
+    {
+        jsonValidate(json);
+    }
+    CATCH_ANY()
+    {
+        // Errors thrown for invalid input are expected
+    }
+    TRY_END();
+
+    // Read the JSON into a variant
+    jsonToVar(json);
+
+    FUNCTION_TEST_RETURN_VOID();
+}

test/fuzz/pack.c

@@ -0,0 +1,107 @@
+/***********************************************************************************************************************************
+Fuzz Pack
+***********************************************************************************************************************************/
+#include <build.h>
+
+#include "common/debug.h"
+#include "common/type/pack.h"
+
+#include "fuzz.h"
+
+/***********************************************************************************************************************************
+Read every field in a container
+***********************************************************************************************************************************/
+static void
+fuzzPackRead(PackRead *const read)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(PACK_READ, read);
+    FUNCTION_TEST_END();
+
+    while (pckReadNext(read))
+    {
+        const unsigned int id = pckReadId(read);
+
+        switch (pckReadType(read))
+        {
+            case pckTypeArray:
+                pckReadArrayBeginP(read, .id = id);
+                fuzzPackRead(read);
+                pckReadArrayEndP(read);
+                break;
+
+            case pckTypeBin:
+                pckReadBinP(read, .id = id);
+                break;
+
+            case pckTypeBool:
+                pckReadBoolP(read, .id = id);
+                break;
+
+            case pckTypeI32:
+                pckReadI32P(read, .id = id);
+                break;
+
+            case pckTypeI64:
+                pckReadI64P(read, .id = id);
+                break;
+
+            case pckTypeMode:
+                pckReadModeP(read, .id = id);
+                break;
+
+            case pckTypeObj:
+                pckReadObjBeginP(read, .id = id);
+                fuzzPackRead(read);
+                pckReadObjEndP(read);
+                break;
+
+            case pckTypePack:
+            {
+                PackRead *const readPack = pckReadPackReadP(read, .id = id);
+
+                fuzzPackRead(readPack);
+                pckReadEndP(readPack);
+                break;
+            }
+
+            case pckTypeStr:
+                pckReadStrP(read, .id = id);
+                break;
+
+            case pckTypeStrId:
+                pckReadStrIdP(read, .id = id);
+                break;
+
+            case pckTypeTime:
+                pckReadTimeP(read, .id = id);
+                break;
+
+            case pckTypeU32:
+                pckReadU32P(read, .id = id);
+                break;
+
+            case pckTypeU64:
+                pckReadU64P(read, .id = id);
+                break;
+        }
+    }
+
+    FUNCTION_TEST_RETURN_VOID();
+}
+
+/**********************************************************************************************************************************/
+void
+fuzzTarget(const Buffer *const input)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(BUFFER, input);
+    FUNCTION_TEST_END();
+
+    PackRead *const read = pckReadNewC(bufPtrConst(input), bufUsed(input));
+
+    fuzzPackRead(read);
+    pckReadEndP(read);
+
+    FUNCTION_TEST_RETURN_VOID();
+}

test/lib/command/code/format.py

@@ -24,6 +24,7 @@ _FILE_EXECUTABLE_LIST = (
     "doc/release.py",
     "build/dist.sh",
     "test/ci.py",
+    "test/fuzz/build.sh",
     "test/smoke.py",
     "test/test.py",
 )

test/lib/command/lint/lint.py

@@ -39,6 +39,7 @@ _LINE_SKIP_LIST = (
     "doc/xml/",  # Documentation, which the coding standards exempt
     "test/certificate/",  # Generated certificates and keys
     "test/data/",  # Test fixtures
+    "test/fuzz/corpus/",  # Fuzz seed inputs
     "test/uncrustify.cfg",  # Written by uncrustify rather than by hand
 )
 

test/meson.build

@@ -10,3 +10,44 @@ test(
     suite: 'smoke',
     timeout: 1800,
 )
+
+####################################################################################################################################
+# Fuzz targets
+####################################################################################################################################
+# Link each target in fuzz with the objects of the pgbackrest binary, except main.c, and the fuzzing engine. The sources must be
+# compiled with the instrumentation flags of the engine, e.g. -fsanitize=fuzzer-no-link.
+if get_option('fuzz-engine') != ''
+    fuzz_src = []
+
+    foreach src : src_pgbackrest
+        if src != 'main.c'
+            fuzz_src += src
+        endif
+    endforeach
+
+    fuzz_object = pgbackrest_exe.extract_objects(fuzz_src)
+
+    foreach fuzz : ['ini', 'json', 'pack']
+        executable(
+            'fuzz-' + fuzz,
+            'fuzz/fuzz.c',
+            'fuzz/' + fuzz + '.c',
+            objects: fuzz_object,
+            include_directories: include_directories('../src'),
+            dependencies: [
+                lib_backtrace,
+                lib_bz2,
+                lib_openssl,
+                lib_lz4,
+                lib_systemd,
+                lib_pq,
+                lib_socket,
+                lib_ssh2,
+                lib_xml,
+                lib_z,
+                lib_zstd,
+            ],
+            link_args: get_option('fuzz-engine').split(),
+        )
+    endforeach
+endif