From dd603ee24e82f1ab488e372a34c18ab304bdcc93 Mon Sep 17 00:00:00 2001 From: jens Steinbacher Date: Tue, 13 Oct 2026 12:42:59 +0000 Subject: [PATCH 01/18] Use streaming jsoncons SAX parsing and test all JSON engines --- README.md | 27 +++ docs/jsoncons-sax.md | 123 ++++++++++ docs/validation-results.md | 161 +++++++++++++ src/jsonbench.c | 42 +++- src/jsonconsparser.cpp | 399 ++++++++++++++++++++++++++++---- src/jsonconsparser.h | 9 + tests/README.md | 78 +++++++ tests/benchmark_jsoncons.py | 287 +++++++++++++++++++++++ tests/benchmark_rusage.c | 47 ++++ tests/jsoncons_api_tests.cpp | 316 +++++++++++++++++++++++++ tests/regression_tests.py | 431 +++++++++++++++++++++++++++++++++++ tests/run_tests.sh | 190 +-------------- 12 files changed, 1873 insertions(+), 237 deletions(-) create mode 100644 docs/jsoncons-sax.md create mode 100644 docs/validation-results.md create mode 100644 tests/README.md create mode 100644 tests/benchmark_jsoncons.py create mode 100644 tests/benchmark_rusage.c create mode 100644 tests/jsoncons_api_tests.cpp create mode 100755 tests/regression_tests.py diff --git a/README.md b/README.md index 6aee8dd..87fde65 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,33 @@ tests/run_tests.sh # all available engines tests/run_tests.sh SIMDJSON # single engine ``` +The runner requires Python 3 (standard library only). Every selected engine +runs the same regression matrix: explicit argument output, valid/invalid JSON, +UTF-8, numeric ranges, size/limit boundaries, silent mode and repeated passes. +Failures and unsupported-limit skips are reported separately; missing expected +output is a failure. A JSONCONS-only run also compares applicable cases to YAJL +when it is installed. Other engines' known defects remain visible as failures. + +The jsoncons adapter uses streaming SAX events, strict JSON syntax and lossless +number events. See [the implementation and compatibility notes](docs/jsoncons-sax.md) +and [validation results](docs/validation-results.md). + +To exercise the jsoncons C interface separately after configuring the project +(use the same include flags as your normal build): + +```bash +g++ -std=c++17 ${CPPFLAGS:-} tests/jsoncons_api_tests.cpp src/jsonconsparser.cpp -o /tmp/jsoncons_api_tests +/tmp/jsoncons_api_tests +``` + +For memory-safety validation, add `-fsanitize=address,undefined +-fno-omit-frame-pointer` to that compile command. For repeatable DOM/SAX/YAJL +measurements, preserve a pre-change executable and run: + +```bash +python3 tests/benchmark_jsoncons.py --baseline /path/to/jsonbench-before --binary src/jsonbench +``` + --- ## TODO diff --git a/docs/jsoncons-sax.md b/docs/jsoncons-sax.md new file mode 100644 index 0000000..c6545cb --- /dev/null +++ b/docs/jsoncons-sax.md @@ -0,0 +1,123 @@ +# jsoncons SAX evaluation + +This change implements the parser evaluation requested in +[ModSecurity issue #3308](https://github.com/owasp-modsecurity/ModSecurity/issues/3308#issuecomment-6142615388). +It does not change ModSecurity or claim integration compatibility with its complete request-body processor. + +## Original behavior and implementation + +The former adapter read an entire file into an `ostringstream`, constructed a +complete `jsoncons::json` DOM, and discarded it. It produced no arguments and +returned unsupported for depth and argument limits. A single-engine test used +itself as reference; missing output and unsupported limits could therefore +appear successful. The combined runner could terminate during `diff | head` +before reporting all failures. + +The new adapter uses public `basic_json_reader`/`json_visitor` APIs (verified with +jsoncons 1.3.2), a streaming binary `ifstream`, and event callbacks. No JSON DOM +or whole-file string is constructed. `read(ec)` includes end-of-input validation +and rejects trailing content. A strict parsing handler rejects comments (the +library's default handler accepts them). Syntax errors include line/column; +I/O, output, allocation and limit failures propagate across the existing C API +without leaking C++ exceptions. Streams and visitor state use RAII. + +The expanded tests exposed a jsoncons 1.3.2 decoding defect: a lone escaped low +surrogate is silently dropped, and a high surrogate followed by a non-low code +unit can be converted into the wrong character. A small public streaming-source +wrapper validates Unicode escape pairing across input buffers before forwarding +them. It performs no JSON grammar parsing or whole-file pass. Raw UTF-8 and the +remaining JSON syntax are validated by jsoncons. + +Each scalar produces one argument, in event order, including duplicate keys; +null produces an empty value. Empty containers produce none. Limits count root +containers as depth 1 and root scalars as depth 0. A depth/argument limit of N +allows exactly N, then rejects the next container/scalar. Direct C callers can +set zero; CLI `-d`/`-a` require decimal integers in 1..UINT_MAX. Every parse call +starts with fresh state and retains the configured limits. Silent mode suppresses +argument writes while enforcing all parsing, count and size checks. + +## Argument names and intentional differences + +The customary YAJL names are retained: `{"a":{"b":1}}` produces `a.b: 1`, +`{"a":[1,2]}` produces `a.a: 1` and `a.a: 2`, and `[1,2]` uses `array.array`. +Array indexes are not added. Nested arrays repeat the enclosing array key. +Duplicate object keys remain separate arguments. + +A structural stack records prefix lengths and parent keys. The shared +`PathHandler` is unsuitable here: it infers parent boundaries from literal dots, +has an off-by-one output-buffer check, and cannot reliably restore parent keys +containing dots. It and the other engines remain unchanged. + +Intentional corrections and numeric normalizations are explicit regression cases: + +- `{"a.b":{"c":1},"d":2}` ends with `d: 2`; YAJL incorrectly emits `a.d: 2`. +- Mixed root arrays, including `[{"x":1},2]`, restore the containing array key: + both adapters produce `array.array.x: 1` then `array.array: 2` in this case. +- Native integers use exact signed/unsigned 64-bit conversions. Oversized + integers arrive as `bigint` string events, preserving all digits, without + conversion to floating point or arbitrary-precision arithmetic. +- `lossless_number(true)` delivers decimal/scientific numbers as `bigdec` + string events. Decimals, exponent magnitude and trailing decimal zeros remain + exact, including `1e9999` and `1e-9999`. These are argument strings, not claims + that a downstream binary float can represent their range. +- jsoncons normalizes integer `-0` to `0` and removes an explicit positive + exponent sign (`1e+03` becomes `1e03`). Negative-zero decimal `-0.0` remains. + Byte compatibility is only asserted for cases without these differences. +- Decoded embedded NULs are written by length instead of truncated like YAJL's + `%s` output. Raw newline/colon/dot characters retain the CLI's ambiguous text + convention; consumers requiring unambiguous framing should use length-aware + callbacks rather than parse this display format. +- jsoncons finalizes documents and rejects truncated input. YAJL's existing + file adapter does not call `yajl_complete_parse`, so it can accept truncation. + Root numbers without a trailing delimiter also lose their argument in YAJL; + jsoncons emits it correctly. These are two separately tested EOF differences. +- The public jsoncons source adaptor accepts a UTF-8 BOM; YAJL rejects it. + This was checked separately and is permitted by RFC 8259's parser guidance. + +## Memory and safety limits + +Keys, full argument paths and decoded values must be shorter than the configured +`JSON_STRING_SIZE` (8192 bytes by default), in both output and silent modes. +Checked dynamic strings avoid fixed-buffer writes. jsoncons buffers an individual +token before visiting it, so its public temporary allocator is additionally +bounded: each token/state allocation may use at most +`4 * JSON_STRING_SIZE + 16384` bytes (48 KiB by default). Excessive strings, +numbers and state growth fail explicitly, including unterminated large tokens. +This budget allows buffer growth and ordinary escaped strings within the value +limit; it is a resource ceiling in addition to semantic limits. + +Memory grows with retained path/depth state and bounded tokenizer storage, rather +than with total document size. There is no fixed whole-file size limit in this +adapter. Configured depth above INT_MAX is capped by jsoncons's int-based depth +option; the temporary state allocation ceiling can reject far smaller extreme +depths too. Deep recursion is avoided by the event reader and explicit stack. + +The other engines have different syntax, range, output and resource behavior. +The regression runner reports those differences as failures where they violate +the required contract; it does not repair their implementations or label absent +argument output as compatible. Missing optional engines remain optional. + +## Verification and integration guidance + +Run the repository build and engine matrix documented in the README. The C API +harness independently checks invalid parameters, setter boundaries, zero limits, +cleanup, error ownership and reuse after failed parses. The benchmark helper +compares a preserved pre-change executable against the SAX executable and YAJL; +its deterministic inputs and raw samples make the measurement reproducible. +See `validation-results.md` for measured results and commands. + +Before adopting jsoncons in ModSecurity: + +1. Pin and test the jsoncons version and strict/lossless options. Test its public + incremental `json_parser` API with actual body chunks and an explicit final + parse step; this adapter currently exposes only a file entry point. +2. Send visitor arguments into ModSecurity's length-aware argument API instead + of stdout. Retain duplicate order, null semantics and both limits. +3. Agree on array names, dotted/empty keys, embedded NULs and numeric lexical + normalizations; replay a real request corpus with explicit expectations. +4. Match body-size, decoded-token, depth and argument budgets to ModSecurity's + configuration. Handle partial events on late parse failure consistently with + request-body error handling; streaming output is not transactional. +5. Measure throughput, allocation, latency and peak memory on representative + concurrent requests, malformed bodies and production compiler/toolchains. + SAX parsing here does not establish streaming JSON-schema validation. diff --git a/docs/validation-results.md b/docs/validation-results.md new file mode 100644 index 0000000..e3faf17 --- /dev/null +++ b/docs/validation-results.md @@ -0,0 +1,161 @@ +# Validation results + +Validated in `/workspace/digitalwave-jsonbench` based on upstream commit +`a89f2e93fb7f1415095d6fb6c1bb5fab6bdc8e6d`. The previous onboarding fork in +`/workspace/jsonbench` was left unchanged. Platform: Debian 13 amd64, GCC 14.2, +jsoncons 1.3.2. Dependencies/tool activation reside outside the checkout under +`/workspace/.jsonbench-tools`; no lockfile or dependency declaration was changed. +Glaze was unavailable and remains optional/unbuilt/unrun. Ten other engines +were discovered and exercised. + +## Build and checks + +From the target checkout, with the existing tool activation sourced: + +```sh +source /workspace/.jsonbench-tools/activate.sh +./autogen.sh +./configure +make -j2 +tests/run_tests.sh JSONCONS +tests/run_tests.sh +``` + +Autotools generation, default configuration and build succeeded. The final +changed parser was compiled with no new warnings. `git diff --check` and shell +syntax checks passed. No ModSecurity source or other parser implementation was +modified. Changes to `jsonbench.c` check unsigned CLI limits and propagate errors +from jsoncons' non-timed output pass with `-n > 1`. + +- JSONCONS selection: **260 passed, 0 failed, 0 skipped**; **6 checked behavioral + differences**, reported separately. Exit 0. 28 explicit cases matched YAJL + byte for byte. Existing fixtures also matched their YAJL reference output. +- Entire ten-engine matrix: **1563 passed, 629 failed, 156 skipped**, plus the + same 6 checked differences. Exit 1, as required. All skips are observed + unsupported-limit warnings; no missing argument output is labelled compatible. +- 56 parser invocations crashed in the full matrix; they were failures, the + runner completed its summary, and core dumps were disabled. No timeouts. +- Opt-in timing (`JSONBENCH_TIMING_RUNS=2`) also passed for JSONCONS; timing + errors would fail the run rather than being ignored. +- A preserved pre-change DOM executable fails **107 required assertions** under + the final JSONCONS matrix (133 passed, 26 unsupported-limit skips, exit 1). + This confirms the new tests detect the former missing-output/limit behavior. +- A separate source copy configured with `--with-jsoncons=no` built successfully + and retained nine engines. Selecting JSONCONS there fails clearly as unavailable. + +The shared matrix contains 34 valid and 41 invalid explicit fixtures, plus +constructed exact/exceeded depth and argument boundaries, string/path boundaries, +2 MiB hostile tokens, a document above the legacy file-size ceiling, invalid CLI +limits, repeated passes and I/O errors. Fixtures are deterministic temporary +files. Golden expectations compare complete bytes, preserving NULs/newlines and +real `Time` keys. Negative checks require a diagnostic and ordinary failure: +signals/timeouts never count as successful rejection or unsupported skips. + +## Results per engine in the full matrix + +The separate JSONCONS/YAJL block adds 28 compatibility passes and 6 checked +DIFFs; it is not included in individual rows below. + +| Engine | Passed | Failed | Skipped | +|---|---:|---:|---:| +| YAJL | 191 | 41 | 0 | +| RAPIDJSON | 191 | 41 | 0 | +| NLOHMANNJSON | 209 | 23 | 0 | +| JSONC | 93 | 113 | 26 | +| JANSSON | 135 | 71 | 26 | +| CJSON | 122 | 84 | 26 | +| JSONCPP | 106 | 100 | 26 | +| JSONCONS | 232 | 0 | 0 | +| SIMDJSON | 108 | 98 | 26 | +| YYJSON | 148 | 58 | 26 | + +The non-jsoncons failures expose existing adapter behavior: numeric reformatting +or loss of range, missing argument output in DOM wrappers, dropped NULs, +incorrect dotted-key restoration, accepted malformed/UTF-8 inputs, missing +finalization, unenforced limits, fixed file buffers and unsafe oversized-key +handling. Individual failures and child exit codes are captured in +`/tmp/jsonbench-root-all-tests.log`. A full replay against the preserved pre-change executable also failed; every +non-jsoncons failure label in the current matrix was present in that baseline. +These counts describe contract checks, not 629 independent bugs. Other engines were retained and their assertions remain +strict; their defects were not patched as part of this jsoncons migration. + +## Memory-safety and API evidence + +A standalone C API harness compiled with `-Wall -Wextra -Werror` passed **220 +checks**. It also passed all 220 under AddressSanitizer + UndefinedBehaviorSanitizer +and leak detection, with no diagnostics: + +```sh +g++ -std=c++17 -Wall -Wextra -Werror -g -O1 \ + -fno-omit-frame-pointer -fsanitize=address,undefined $CPPFLAGS \ + tests/jsoncons_api_tests.cpp src/jsonconsparser.cpp -o /tmp/jsoncons-api-tests +ASAN_OPTIONS=detect_leaks=1:halt_on_error=1 \ +UBSAN_OPTIONS=halt_on_error=1 /tmp/jsoncons-api-tests +``` + +The complete CLI matrix was additionally run against a separately built +JSONCONS-only ASan/UBSan executable: **226 passed, 0 failed, 40 skipped**, exit 0, +no sanitizer diagnostics. Those 40 skips are YAJL comparisons unavailable in +that isolated build, rather than untested jsoncons functionality. The original +all-engine executable was not replaced for this check. Sanitizer build flags: +`-g -O1 -fno-omit-frame-pointer -fno-pie -fsanitize=address,undefined` and linking +with `-no-pie -fsanitize=address,undefined`. + +Independent source review found no remaining material lifetime, allocation, +conversion or buffer issue in the changed implementation. A separate 34-case +Unicode probe verified valid/invalid pairs at 15 offsets around the 16 KiB +buffer boundary, escaped backslashes/quotes, U+10000 and U+10FFFF. Resource +ceilings, partial output on late failures and version-specific surrogate +validation remain explicit integration considerations. + +## Performance measurement + +```sh +python3 tests/benchmark_jsoncons.py \ + --baseline /workspace/.jsonbench-tools/jsonbench-dom-baseline \ + --binary src/jsonbench --output /tmp/jsonbench-final-benchmark.json +``` + +One warmup then five fresh-process samples per measured variant; table timing is +CLI parser mean ± sample standard deviation, in milliseconds. Memory is mean +child-process peak RSS in KiB, measured with a compiled Linux `wait4` helper. +Input: root array of identical short three-field objects, silent output, argument +limit set exactly to the generated leaf count. Effective production optimization +was O2. Binary/config/input hashes and raw samples are in the JSON report; hashes +were checked unchanged during the measurement. + +| Actual input bytes | Variant | Parser ms, mean ± stddev | Mean peak RSS, KiB | +|---:|---|---:|---:| +| 409,564 | jsoncons_dom | 4.477 ± 0.648 | 5330 | +| 409,564 | jsoncons_sax | 2.932 ± 0.079 | 2671 | +| 409,564 | yajl | 1.498 ± 0.208 | 2734 | +| 5,242,854 | yajl | Unrun: configured file limit | — | +| 5,242,854 | jsoncons_dom | 69.876 ± 8.417 | 38976 | +| 5,242,854 | jsoncons_sax | 41.717 ± 5.489 | 2673 | +| 33,554,368 | yajl | Unrun: configured file limit | — | +| 33,554,368 | jsoncons_dom | 363.273 ± 21.097 | 228209 | +| 33,554,368 | jsoncons_sax | 240.755 ± 22.905 | 2649 | + +On this synthetic cached-file workload SAX used substantially less peak memory +than the previous DOM implementation and lower parser time. At 32 MiB the +measured values were approximately 363 ms/223 MiB for DOM and 241 ms/2.6 MiB for +SAX. YAJL was faster on the common smaller fixture; its configured 512 KiB file +buffer prevented larger comparisons. The old DOM adapter merely parsed/discarded +its tree, while SAX also performs argument-path/count/size checks and the Unicode +escape guard. No general ModSecurity throughput or concurrency claim follows. + +## Changed and added files + +Modified: `src/jsonconsparser.cpp`, `src/jsonconsparser.h`, `src/jsonbench.c`, +`tests/run_tests.sh`, `README.md`. + +Added: `tests/regression_tests.py`, `tests/jsoncons_api_tests.cpp`, +`tests/benchmark_jsoncons.py`, `tests/benchmark_rusage.c`, `tests/README.md`, +`docs/jsoncons-sax.md`, `docs/validation-results.md`. + +Generated local Autotools artifacts (`build-aux/`, `config.h`, configure backups, +and ignored object/build files) are not source changes and should not be committed. +See [the SAX notes](jsoncons-sax.md) for the six explicit YAJL differences, +optional BOM behavior, remaining resource/API limits and ModSecurity integration +recommendations. This remains a file-parser evaluation, not an implemented +ModSecurity chunked-body or schema-validation integration. diff --git a/src/jsonbench.c b/src/jsonbench.c index 8a2d06c..12fdbdf 100644 --- a/src/jsonbench.c +++ b/src/jsonbench.c @@ -18,6 +18,7 @@ #include #include #include +#include #ifndef LIMIT_ARG_NUM #define LIMIT_ARG_NUM 500 @@ -57,6 +58,23 @@ static void print_stats(const double *times, int n) { static char available_engines[16][20] = {""}; static int engine_count = 0; +/* Do not let atoi overflow, accept a prefix, or turn negatives into unsigned + * limits. The parser interfaces take double, which exactly represents UINT_MAX. + */ +static int parse_limit(const char *text, unsigned int *value) { + if (text == NULL || *text == '\0') return 0; + for (const char *p = text; *p; ++p) { + if (*p < '0' || *p > '9') return 0; + } + errno = 0; + char *end = NULL; + unsigned long parsed = strtoul(text, &end, 10); + if (errno == ERANGE || *end != '\0' || parsed == 0 || parsed > UINT_MAX) + return 0; + *value = (unsigned int)parsed; + return 1; +} + static void showhelp(void) { printf("Use: jsonbench [OPTIONS]\n\n"); printf("OPTIONS:\n"); @@ -154,18 +172,16 @@ int main(int argc, char ** argv) { } break; case 'a': - arg_limit = atoi(optarg); - if (arg_limit == 0 || arg_limit > (unsigned int)UINT_MAX) { - fprintf(stderr, "Ohh... Try to pass for '-a' an integer between 0 and %u\n", + if (!parse_limit(optarg, &arg_limit)) { + fprintf(stderr, "Pass for '-a' an integer between 1 and %u\n", (unsigned int)UINT_MAX); free(jsonengine); return EXIT_FAILURE; } break; case 'd': - depth_limit = atoi(optarg); - if (depth_limit == 0 || depth_limit > (unsigned int)UINT_MAX) { - fprintf(stderr, "Ohh... Try to pass for '-d' an integer between 0 and %u\n", + if (!parse_limit(optarg, &depth_limit)) { + fprintf(stderr, "Pass for '-d' an integer between 1 and %u\n", (unsigned int)UINT_MAX); free(jsonengine); return EXIT_FAILURE; @@ -759,13 +775,21 @@ int main(int argc, char ** argv) { if (n > 1 && !silence) { jsoncons_parser *json = NULL; - jsoncons_json_init(&json, &error_msg); - if (json != NULL) { + rc = jsoncons_json_init(&json, &error_msg); + if (rc == 0 && json != NULL) { jsoncons_set_max_depth(json, depth_limit); jsoncons_set_max_arg_num(json, arg_limit); jsoncons_set_silence(json, 0); - jsoncons_parse_file(json, jsonfile, &error_msg); + rc = jsoncons_parse_file(json, jsonfile, &error_msg); jsoncons_json_cleanup(json); + } else { + rc = 1; + } + if (rc != 0) { + if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } + free(times); + free(jsonengine); + return EXIT_FAILURE; } } } diff --git a/src/jsonconsparser.cpp b/src/jsonconsparser.cpp index 6f35840..34e7fe6 100644 --- a/src/jsonconsparser.cpp +++ b/src/jsonconsparser.cpp @@ -2,84 +2,387 @@ #if HAVE_JSONCONS +#include +#include +#include +#include #include -#include +#include +#include #include -#include +#include #include -#include +#include +#include +#include + +#include +#include +#include struct jsoncons_parser { - int silence; + unsigned int max_depth = 500; + unsigned int max_args = 500; + bool silence = false; }; -extern "C" int jsoncons_json_init(jsoncons_parser **parser, char **error_msg) { - if (parser == nullptr || error_msg == nullptr) { - return -1; +namespace { + +// jsoncons 1.3.2 silently drops lone escaped low surrogates and does not check +// that the second code unit of a pair is a low surrogate. Validate just this +// lexical boundary while forwarding its public stream source, including pairs +// split over buffers. All JSON grammar and raw UTF-8 validation stay in jsoncons. +class UnicodeSource { + enum class State { outside, text, escape, hex, pair_slash, pair_u }; + jsoncons::stream_source source_; + State state_ = State::outside; + unsigned int codepoint_ = 0; + int digits_ = 0; + bool low_required_ = false; + std::size_t line_ = 1; + std::size_t column_ = 0; + + [[noreturn]] void invalid() const { + throw std::invalid_argument("Invalid Unicode surrogate escape at line " + + std::to_string(line_) + ", column " + std::to_string(column_)); } - try { - *parser = new jsoncons_parser{}; - } catch (const std::bad_alloc&) { - *parser = nullptr; - *error_msg = strdup("Unable to allocate jsoncons parser"); - return 1; + void check(unsigned char c) { + if (c == '\n') { ++line_; column_ = 0; } else { ++column_; } + switch (state_) { + case State::outside: + if (c == '"') state_ = State::text; + break; + case State::text: + if (c == '"') state_ = State::outside; + else if (c == '\\') state_ = State::escape; + break; + case State::escape: + if (c == 'u') { + state_ = State::hex; + codepoint_ = 0; + digits_ = 0; + } else { + state_ = State::text; + } + break; + case State::hex: { + unsigned int digit; + if (c >= '0' && c <= '9') digit = c - '0'; + else if (c >= 'a' && c <= 'f') digit = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') digit = c - 'A' + 10; + else invalid(); + codepoint_ = codepoint_ * 16 + digit; + if (++digits_ != 4) break; + if (low_required_) { + if (codepoint_ < 0xdc00 || codepoint_ > 0xdfff) invalid(); + low_required_ = false; + state_ = State::text; + } else if (codepoint_ >= 0xd800 && codepoint_ <= 0xdbff) { + low_required_ = true; + state_ = State::pair_slash; + } else { + if (codepoint_ >= 0xdc00 && codepoint_ <= 0xdfff) invalid(); + state_ = State::text; + } + break; + } + case State::pair_slash: + if (c != '\\') invalid(); + state_ = State::pair_u; + break; + case State::pair_u: + if (c != 'u') invalid(); + state_ = State::hex; + codepoint_ = 0; + digits_ = 0; + break; + } } - *error_msg = nullptr; - return 0; -} + public: + using value_type = char; + explicit UnicodeSource(std::istream& input) : source_(input) {} + UnicodeSource(UnicodeSource&&) = default; + bool eof() const { return source_.eof(); } + bool is_error() const { return source_.is_error(); } + jsoncons::span read_buffer() { + const auto data = source_.read_buffer(); + for (const unsigned char c : data) check(c); + return data; + } +}; -extern "C" int jsoncons_parse_file(jsoncons_parser *parser, const char *filename, char **error_msg) { - (void)parser; - if (filename == nullptr || error_msg == nullptr) { - return -1; +// jsoncons materializes one decoded token before delivering its event. Bound +// its temporary allocations as well as the visitor lengths, so an unterminated +// multi-megabyte string/number cannot consume document-sized memory first. +// Leave room for geometric growth and the parser's structural state stack. +template +class TemporaryAllocator { + public: + using value_type = T; + TemporaryAllocator() = default; + template TemporaryAllocator(const TemporaryAllocator&) {} + T* allocate(std::size_t count) { + constexpr std::size_t budget = + static_cast(JSON_STRING_SIZE) * 4 + 16384; + if (count > budget / sizeof(T)) + throw std::length_error("jsoncons token/state allocation limit exceeded"); + return std::allocator{}.allocate(count); + } + void deallocate(T* data, std::size_t count) noexcept { + std::allocator{}.deallocate(data, count); } + template bool operator==(const TemporaryAllocator&) const { return true; } + template bool operator!=(const TemporaryAllocator&) const { return false; } +}; - std::ifstream input(filename); - if (!input.is_open()) { - *error_msg = strdup("Unable to open JSON file"); - return 2; +// Keep structural boundaries rather than finding them by searching for dots. +// PathHandler cannot distinguish literal dots/empty keys from nesting, and its +// fixed buffers have unsafe boundary checks. This stack follows the same usual +// dotted naming convention without copying those limitations. +class ArgumentVisitor final : public jsoncons::default_json_visitor { + struct Frame { + std::size_t prefix_length; + std::string key; + }; + + const jsoncons_parser& settings_; + std::vector stack_; + std::string prefix_; + std::string key_; + unsigned int arguments_ = 0; + const char* error_ = nullptr; + + void fail(const char* message, std::error_code& ec) { + error_ = message; + // Visitor return values alone do not abort jsoncons::json_reader. + ec = std::make_error_code(std::errc::invalid_argument); } - std::ostringstream oss; - oss << input.rdbuf(); + void begin(bool array, std::error_code& ec) { + if (stack_.size() >= settings_.max_depth) { + fail("JSON depth limit exceeded", ec); + return; + } + const bool root_array = array && stack_.empty(); + const std::size_t length = root_array ? 5 : + prefix_.size() + (prefix_.empty() ? 0 : 1) + key_.size(); + if (length >= JSON_STRING_SIZE) { + fail("Argument name too long", ec); + return; + } + stack_.push_back({prefix_.size(), key_}); + if (root_array) { + prefix_ = "array"; + key_ = "array"; + } else { + if (!prefix_.empty()) prefix_ += '.'; + prefix_ += key_; + } + } - try { - auto parsed = jsoncons::json::parse(oss.str()); - (void)parsed; - } catch (const std::exception &e) { - *error_msg = strdup(e.what()); - return 2; + void end() { + // The JSON reader guarantees properly paired begin/end events. + prefix_.resize(stack_.back().prefix_length); + key_ = std::move(stack_.back().key); + stack_.pop_back(); } - return 0; -} + void argument(const char* value, std::size_t length, std::error_code& ec) { + if (arguments_ >= settings_.max_args) { + fail("Argument number limit exceeded", ec); + return; + } + const std::size_t name_length = prefix_.size() + + (prefix_.empty() ? 0 : 1) + key_.size(); + // Silent parsing must enforce the same validation and size limits. + if (name_length >= JSON_STRING_SIZE) { + fail("Argument name too long", ec); + return; + } + if (length >= JSON_STRING_SIZE) { + fail("Argument value too long", ec); + return; + } + ++arguments_; + if (settings_.silence) return; + + // Length-aware writes preserve decoded embedded NULs and Unicode. + bool ok = true; + if (!prefix_.empty()) { + ok = std::fwrite(prefix_.data(), 1, prefix_.size(), stdout) == prefix_.size(); + ok = (std::fputc('.', stdout) != EOF) && ok; + } + ok = (std::fwrite(key_.data(), 1, key_.size(), stdout) == key_.size()) && ok; + ok = (std::fwrite(": ", 1, 2, stdout) == 2) && ok; + ok = (std::fwrite(value, 1, length, stdout) == length) && ok; + ok = (std::fputc('\n', stdout) != EOF) && ok; + if (!ok) fail("Unable to write JSON argument output", ec); + } + + JSONCONS_VISITOR_RETURN_TYPE visit_begin_object(jsoncons::semantic_tag, + const jsoncons::ser_context&, std::error_code& ec) override { + begin(false, ec); + JSONCONS_VISITOR_RETURN; + } + JSONCONS_VISITOR_RETURN_TYPE visit_begin_array(jsoncons::semantic_tag, + const jsoncons::ser_context&, std::error_code& ec) override { + begin(true, ec); + JSONCONS_VISITOR_RETURN; + } + JSONCONS_VISITOR_RETURN_TYPE visit_end_object(const jsoncons::ser_context&, + std::error_code&) override { + end(); + JSONCONS_VISITOR_RETURN; + } + JSONCONS_VISITOR_RETURN_TYPE visit_end_array(const jsoncons::ser_context&, + std::error_code&) override { + end(); + JSONCONS_VISITOR_RETURN; + } + JSONCONS_VISITOR_RETURN_TYPE visit_key(const string_view_type& name, + const jsoncons::ser_context&, std::error_code& ec) override { + if (name.size() >= JSON_STRING_SIZE) { + fail("Argument name too long", ec); + } else { + key_.assign(name.data(), name.size()); + } + JSONCONS_VISITOR_RETURN; + } + JSONCONS_VISITOR_RETURN_TYPE visit_null(jsoncons::semantic_tag, + const jsoncons::ser_context&, std::error_code& ec) override { + argument("", 0, ec); + JSONCONS_VISITOR_RETURN; + } + JSONCONS_VISITOR_RETURN_TYPE visit_bool(bool value, jsoncons::semantic_tag, + const jsoncons::ser_context&, std::error_code& ec) override { + argument(value ? "true" : "false", value ? 4 : 5, ec); + JSONCONS_VISITOR_RETURN; + } + JSONCONS_VISITOR_RETURN_TYPE visit_string(const string_view_type& value, + jsoncons::semantic_tag, const jsoncons::ser_context&, + std::error_code& ec) override { + // lossless_number delivers decimals and oversized integers as tagged + // strings, so no conversion through double (or truncation) is needed. + argument(value.data(), value.size(), ec); + JSONCONS_VISITOR_RETURN; + } + JSONCONS_VISITOR_RETURN_TYPE visit_int64(int64_t value, jsoncons::semantic_tag, + const jsoncons::ser_context&, std::error_code& ec) override { + const std::string text = std::to_string(value); + argument(text.data(), text.size(), ec); + JSONCONS_VISITOR_RETURN; + } + JSONCONS_VISITOR_RETURN_TYPE visit_uint64(uint64_t value, jsoncons::semantic_tag, + const jsoncons::ser_context&, std::error_code& ec) override { + const std::string text = std::to_string(value); + argument(text.data(), text.size(), ec); + JSONCONS_VISITOR_RETURN; + } + JSONCONS_VISITOR_RETURN_TYPE visit_double(double, jsoncons::semantic_tag, + const jsoncons::ser_context&, std::error_code& ec) override { + // Defensive: read options must prevent binary floating conversion. + fail("Unexpected non-lossless JSON number event", ec); + JSONCONS_VISITOR_RETURN; + } + + public: + explicit ArgumentVisitor(const jsoncons_parser& settings) : settings_(settings) {} + const char* error() const { return error_; } +}; -extern "C" int jsoncons_set_max_depth(jsoncons_parser *parser, double max_depth) { - if (parser == nullptr) { +int set_limit(jsoncons_parser* parser, double value, unsigned int* target) { + if (parser == nullptr || !std::isfinite(value) || value < 0 || + value > std::numeric_limits::max() || std::floor(value) != value) { return -1; } - (void)max_depth; - return 1; + *target = static_cast(value); + return 0; } -extern "C" int jsoncons_set_max_arg_num(jsoncons_parser *parser, double max_arg_num) { - if (parser == nullptr) { - return -1; +} // namespace + +extern "C" int jsoncons_json_init(jsoncons_parser** parser, char** error_msg) { + if (error_msg == nullptr) return -1; + *error_msg = nullptr; + if (parser == nullptr) return -1; + *parser = nullptr; + try { + *parser = new jsoncons_parser{}; + return 0; + } catch (const std::exception& e) { + *error_msg = strdup(e.what()); + return 1; + } catch (...) { + *error_msg = strdup("Unable to initialize jsoncons parser"); + return 1; } - (void)max_arg_num; - return 1; } -extern "C" int jsoncons_set_silence(jsoncons_parser *parser, int silence) { - if (parser != nullptr) { - parser->silence = silence; +extern "C" int jsoncons_parse_file(jsoncons_parser* parser, const char* filename, + char** error_msg) { + if (error_msg == nullptr) return -1; + *error_msg = nullptr; + if (parser == nullptr || filename == nullptr) return -1; + + // Keep all potentially throwing stream/library/visitor operations inside + // the C boundary; stream, reader and visitor release storage on all exits. + try { + std::ifstream input(filename, std::ios::binary); + if (!input.is_open()) { + *error_msg = strdup("Unable to open JSON file"); + return 2; + } + ArgumentVisitor visitor(*parser); + jsoncons::json_options options; + options.lossless_number(true).err_handler(jsoncons::strict_json_parsing()); + // jsoncons uses int for its own nesting limit. For larger requested + // values its representable ceiling is an additional safety limit. + options.max_nesting_depth(static_cast( + parser->max_depth > static_cast(INT_MAX) ? + INT_MAX : parser->max_depth)); + jsoncons::basic_json_reader> reader(input, visitor, options); + std::error_code ec; + reader.read(ec); // Includes finalization and checks trailing content. + if (ec) { + std::string message = visitor.error() ? visitor.error() : ec.message(); + message += " at line " + std::to_string(reader.line()) + + ", column " + std::to_string(reader.column()); + *error_msg = strdup(message.c_str()); + return 2; + } + if (input.bad()) { + *error_msg = strdup("Error reading JSON file"); + return 2; + } + if (!parser->silence && std::fflush(stdout) != 0) { + *error_msg = strdup("Unable to write JSON argument output"); + return 2; + } + return 0; + } catch (const std::exception& e) { + *error_msg = strdup(e.what()); + return 2; + } catch (...) { + *error_msg = strdup("Unexpected jsoncons parser failure"); + return 2; } - return 0; } -extern "C" int jsoncons_json_cleanup(jsoncons_parser *parser) { +extern "C" int jsoncons_set_max_depth(jsoncons_parser* parser, double max_depth) { + return set_limit(parser, max_depth, parser ? &parser->max_depth : nullptr); +} +extern "C" int jsoncons_set_max_arg_num(jsoncons_parser* parser, double max_args) { + return set_limit(parser, max_args, parser ? &parser->max_args : nullptr); +} +extern "C" int jsoncons_set_silence(jsoncons_parser* parser, int silence) { + if (parser == nullptr) return -1; + parser->silence = silence != 0; + return 0; +} +extern "C" int jsoncons_json_cleanup(jsoncons_parser* parser) { delete parser; return 0; } diff --git a/src/jsonconsparser.h b/src/jsonconsparser.h index b497901..20d375e 100644 --- a/src/jsonconsparser.h +++ b/src/jsonconsparser.h @@ -11,6 +11,15 @@ extern "C" { typedef struct jsoncons_parser jsoncons_parser; +/* Zero denotes success; nonzero denotes failure. Parse/init errors, when + * available, are malloc-owned strings that the caller must free. All C entry + * points contain C++ exceptions. Cleanup accepts NULL; parsing starts fresh + * on every call while retaining the configured limits and silent setting. + * Limits are nonnegative integral doubles <= UINT_MAX. A zero limit allows + * no containers (depth) or no scalar arguments (argument count), respectively. + * CLI limits must be positive. Root containers have depth 1, root scalars 0. + * Null is an empty argument value; every scalar counts, including null/bool. + */ int jsoncons_json_init(jsoncons_parser **parser, char **error_msg); int jsoncons_parse_file(jsoncons_parser *parser, const char *filename, char **error_msg); int jsoncons_set_max_depth(jsoncons_parser *parser, double max_depth); diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..877cbea --- /dev/null +++ b/tests/README.md @@ -0,0 +1,78 @@ +# Regression tests + +Build using the repository's usual `./autogen.sh`, `./configure`, and `make` +commands, then run: + +```sh +tests/run_tests.sh # every available engine, the same required matrix +tests/run_tests.sh JSONCONS # one engine; still compare with YAJL if available +``` + +The existing shell entry point uses `regression_tests.py`, a Python 3 standard +library helper. It writes deterministic temporary fixtures and deletes them +afterwards. No external test framework is required. All parser subprocesses +have a 15 second timeout and disabled core dumps. A parser crash or timeout is +a failure, rather than successful rejection of invalid JSON. + +The matrix applies to **every selected engine**. It checks exact argument names, +byte values, repeated keys, root primitives, nested arrays and objects, empty +containers, booleans/null, Unicode, embedded NUL, escaped characters, numeric +boundaries and precision, malformed JSON/UTF-8, depth and argument boundaries, +long paths/keys/values, huge tokens, a document larger than the legacy file +buffer, silent mode, repeated benchmark iterations, CLI limit validation, file +errors and output errors. Existing fixtures and cross-engine comparisons remain +included. New expected outputs are explicit data in the helper, not generated +by the parser under test. Argument counts are also tested by running exactly at +the leaf count and one below it. Boolean and null leaves count as arguments; +empty containers do not. + +A parse-only adapter that emits no arguments **fails** output tests. An adapter +that explicitly warns that a particular limit is unsupported can skip that +limit check; it still runs parsing, output, numeric and malformed-input tests. +Unavailable YAJL causes a clearly labelled comparison skip. Single-engine runs +never compare the engine with itself. Only byte equality after checking the +expected outputs is reported as compatibility. Known differences are separately +reported as `DIFF`, not as compatibility passes. Full-engine runs expose existing +adapter defects and return nonzero when any required assertion fails. + +The required numeric contract preserves exact integer precision (including +values outside signed/unsigned 64-bit ranges) and decimal/exponent precision. +jsoncons' public lossless-number API normalizes integer `-0` to `0` and removes +the optional exponent `+`; those two differences have explicit expected results. +Other engines are checked against the original numerical spelling. Known YAJL +path-restoration, embedded-NUL and root-number-at-EOF differences are measured +separately; when YAJL itself is selected it must still satisfy the intended safe +path/output contract and fails those affected tests. No output-wide normalization +is applied to comparisons. The helper removes only the CLI timing record at its +defined position, preserving real blank lines and `Time` argument data. + +To test another built executable or reproduce baseline failures: + +```sh +JSONBENCH_BINARY=/absolute/path/to/jsonbench tests/run_tests.sh JSONCONS +``` + +Timing is opt-in, so regression runs do not spend time benchmarking every small +fixture. Timing failures also affect the test exit status: + +```sh +JSONBENCH_TIMING_RUNS=20 tests/run_tests.sh JSONCONS +``` + +`jsoncons_api_tests.cpp` separately tests the C-compatible API, including null +parameters, limit validation, zero-limit semantics, cleanup, error ownership, +parser reuse after success/failure, and settings preserved after invalid setters. +After configuring with JSONCONS enabled, compile and run it with the same header +paths used for the main build: + +```sh +g++ -std=c++17 -Wall -Wextra -Werror $CPPFLAGS \ + tests/jsoncons_api_tests.cpp src/jsonconsparser.cpp -o /tmp/jsoncons-api-tests +/tmp/jsoncons-api-tests +``` + +`CPPFLAGS` must include the installed jsoncons include path if it is outside the +compiler's standard paths. Add `-fsanitize=address,undefined +-fno-omit-frame-pointer` to this command to run the harness with sanitizers. +The API harness returns nonzero on failure and returns 77 if configured without +JSONCONS; that is unavailable coverage, not a passing test. diff --git a/tests/benchmark_jsoncons.py b/tests/benchmark_jsoncons.py new file mode 100644 index 0000000..9815ec3 --- /dev/null +++ b/tests/benchmark_jsoncons.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +"""Measure JSONCONS SAX, an optional preserved DOM binary, and available YAJL. + +Example: + python3 tests/benchmark_jsoncons.py --baseline /path/to/old/jsonbench \ + --binary src/jsonbench --output /tmp/jsoncons-benchmark.json + +The synthetic workload is a root array of short, three-field objects. It is +intended for a reproducible local comparison, not a ModSecurity performance +claim. The original jsoncons DOM adapter performs no argument/path/limit work. +""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import shlex +import signal +import statistics +import subprocess +import sys +import tempfile +import time + + +REPOSITORY = Path(__file__).resolve().parent.parent +ITEM = b'{"name":"abcdefghijklmnopqrstuvwxyz0123456789","id":123456789,"active":true}' +UINT_MAX = 4294967295 + + +def positive_int(text): + value = int(text) + if value <= 0: + raise argparse.ArgumentTypeError("must be a positive integer") + return value + + +def size_bytes(text): + match = re.fullmatch(r"([0-9]+)(B|K|KB|KiB|M|MB|MiB|G|GB|GiB)?", text, re.IGNORECASE) + if not match: + raise argparse.ArgumentTypeError("use bytes or a size such as 400KiB, 5MiB, or 32MiB") + suffix = (match.group(2) or "B").upper() + power = {"B": 0, "K": 1, "KB": 1, "KIB": 1, + "M": 2, "MB": 2, "MIB": 2, "G": 3, "GB": 3, "GIB": 3}[suffix] + value = int(match.group(1)) * 1024 ** power + if value < len(ITEM) + 2: + raise argparse.ArgumentTypeError("size must fit at least one benchmark object") + if ((value - 1) // (len(ITEM) + 1)) * 3 > UINT_MAX: + raise argparse.ArgumentTypeError("generated argument count would exceed the CLI limit") + return value + + +def file_hash(path): + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def executable(parser, path, label): + path = path.expanduser().resolve() + if not path.is_file() or not os.access(path, os.X_OK): + hint = ("preserve the original executable before rebuilding, then pass --baseline PATH" + if label == "baseline" else "run ./autogen.sh, ./configure, and make first") + parser.error(f"{label} executable is missing or not executable: {path}; {hint}") + return path + + +def available_engines(binary): + result = subprocess.run([str(binary), "-h"], stdin=subprocess.DEVNULL, + capture_output=True, text=True, check=False) + if result.returncode: + raise RuntimeError(f"{binary} -h failed: {result.stderr.strip()}") + return set(re.findall(r"^\s*- ([A-Z][A-Z0-9]+)\s*$", result.stdout, re.MULTILINE)) + + +def configured_buffer(path): + if not path.is_file(): + raise RuntimeError(f"configured header missing: {path}; run ./configure or pass --config PATH") + text = path.read_text() + match = re.search(r"^\s*#\s*define\s+FILE_BUFFER_SIZE\s+([0-9]+)[uUlL]*\s*$", text, re.MULTILINE) + if not match or int(match.group(1)) < 2: + raise RuntimeError(f"cannot determine FILE_BUFFER_SIZE from {path}") + return int(match.group(1)) + + +def dataset(directory, target): + count = (target - 1) // (len(ITEM) + 1) + path = directory / f"input-{target}.json" + digest = hashlib.sha256() + with path.open("wb") as stream: + def write(block): + stream.write(block) + digest.update(block) + + write(b"[" + ITEM) + remaining = count - 1 + block = (b"," + ITEM) * 4096 + while remaining >= 4096: + write(block) + remaining -= 4096 + write((b"," + ITEM) * remaining) + write(b"]") + return path, {"target_bytes": target, "bytes": path.stat().st_size, + "objects": count, "arguments": count * 3, "sha256": digest.hexdigest()} + + +def invoke(helper, binary, engine, path, argument_count, timeout): + command = [str(helper), str(binary), "-s", "-a", str(argument_count), + "-n", "1", "-e", engine, str(path)] + start = time.perf_counter() + process = subprocess.Popen(command, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=True) + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + stdout, stderr = process.communicate() + return {"status": "failed", "reason": f"timeout after {timeout} seconds", + "exit_code": process.returncode, + "stdout": stdout.decode(errors="replace"), "stderr": stderr.decode(errors="replace")} + except BaseException: + # Ctrl-C must also stop the helper and its parser subprocess. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate() + raise + wall = time.perf_counter() - start + stdout = stdout.decode(errors="replace") + stderr = stderr.decode(errors="replace") + timing = re.search(r"Time:\s+([0-9.eE+-]+)\s+sec", stdout) + usage = re.search(r"^JSONBENCH_RUSAGE (\{.*\})$", stderr, re.MULTILINE) + if process.returncode or not timing or not usage: + return {"status": "failed", "exit_code": process.returncode, + "reason": "parser failed or required timing/resource output is missing", + "stdout": stdout, "stderr": stderr} + result = {"status": "ok", "parser_seconds": float(timing.group(1)), "wall_seconds": wall} + result.update(json.loads(usage.group(1))) + diagnostics = re.sub(r"^JSONBENCH_RUSAGE .*\n?", "", stderr, flags=re.MULTILINE).strip() + if diagnostics: + result["stderr"] = diagnostics + return result + + +def summary(samples): + result = {} + for metric in ("parser_seconds", "wall_seconds", "peak_rss_kib"): + values = [sample[metric] for sample in samples] + result[metric] = {"mean": statistics.mean(values), + "sample_stdev": statistics.stdev(values) if len(values) > 1 else 0, + "min": min(values), "max": max(values)} + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--binary", type=Path, default=REPOSITORY / "src/jsonbench") + parser.add_argument("--baseline", type=Path, help="preserved executable containing the original DOM implementation") + parser.add_argument("--config", type=Path, default=REPOSITORY / "config.h", + help="configured header used to build --binary, for YAJL's file buffer limit") + parser.add_argument("--samples", type=positive_int, default=5) + parser.add_argument("--warmups", type=positive_int, default=1) + parser.add_argument("--sizes", type=size_bytes, nargs="+", default=None) + parser.add_argument("--timeout", type=positive_int, default=60, help="seconds per process") + parser.add_argument("--output", type=Path, help="write JSON report here instead of stdout") + args = parser.parse_args() + if platform.system() != "Linux": + parser.error("the peak-RSS helper currently requires Linux (ru_maxrss is reported in KiB)") + + binary = executable(parser, args.binary, "SAX") + baseline = executable(parser, args.baseline, "baseline") if args.baseline else None + config = args.config.expanduser().resolve() + try: + file_buffer_size = configured_buffer(config) + engines = available_engines(binary) + if "JSONCONS" not in engines: + raise RuntimeError(f"JSONCONS is unavailable in {binary}; configure with jsoncons headers installed") + if baseline and "JSONCONS" not in available_engines(baseline): + raise RuntimeError(f"JSONCONS is unavailable in the baseline executable {baseline}") + except (OSError, RuntimeError) as error: + parser.error(str(error)) + + common_size = min(400 * 1024, file_buffer_size - 1) + sizes = args.sizes or [max(common_size, len(ITEM) + 2), 5 * 1024 * 1024, 32 * 1024 * 1024] + compiler = shlex.split(os.environ.get("CC", "cc")) + if not compiler: + parser.error("CC is empty; set CC to a C compiler or leave it unset to use cc") + + artifacts = {"binary": {"path": str(binary), "sha256": file_hash(binary)}, + "config": {"path": str(config), "sha256": file_hash(config)}} + if baseline: + artifacts["baseline"] = {"path": str(baseline), "sha256": file_hash(baseline)} + report = {"schema_version": 1, "platform": platform.platform(), "artifacts": artifacts, + "method": {"samples": args.samples, "warmups": args.warmups, + "cli": "-s -a GENERATED_ARGUMENT_COUNT -n 1 -e ENGINE INPUT", + "input": "root array of identical objects: short ASCII string, exact integer, boolean", + "timing": "CLI parser elapsed seconds; independent fresh processes after warmups", + "rss": "Linux wait4 child peak RSS in KiB via a small compiled C launcher", + "file_buffer_size": file_buffer_size, + "yajl_maximum_file_bytes": file_buffer_size - 1, + "compiler": compiler, + "caveat": "Original DOM adapter only parses/discards a DOM. SAX also builds argument paths and enforces argument/size limits. Synthetic cached-file workload; no ModSecurity performance conclusion."}, + "inputs": [], "runs": []} + failed = False + with tempfile.TemporaryDirectory(prefix="jsonbench-benchmark-") as temp: + directory = Path(temp) + helper = directory / "benchmark-rusage" + compile_command = compiler + ["-O2", "-std=c11", "-Wall", "-Wextra", + str(Path(__file__).with_name("benchmark_rusage.c")), "-o", str(helper)] + try: + compile_result = subprocess.run(compile_command, capture_output=True, text=True, check=False) + except OSError as error: + parser.error(f"cannot run C compiler {compiler[0]}: {error}") + if compile_result.returncode: + parser.error("cannot compile peak-RSS helper: " + compile_result.stderr.strip()) + + for target in sizes: + path, info = dataset(directory, target) + report["inputs"].append(info) + variants = [("jsoncons_sax", binary, "JSONCONS")] + if baseline: + variants.insert(0, ("jsoncons_dom", baseline, "JSONCONS")) + else: + report["runs"].append({"variant": "jsoncons_dom", "input_bytes": info["bytes"], + "status": "skipped", "reason": "no baseline supplied; pass --baseline /path/to/preserved/executable"}) + if "YAJL" not in engines: + report["runs"].append({"variant": "yajl", "input_bytes": info["bytes"], + "status": "skipped", "reason": "YAJL is unavailable in the configured binary"}) + elif info["bytes"] > file_buffer_size - 1: + report["runs"].append({"variant": "yajl", "input_bytes": info["bytes"], + "status": "skipped", "reason": f"input exceeds configured YAJL file limit of {file_buffer_size - 1} bytes"}) + else: + variants.append(("yajl", binary, "YAJL")) + + for name, target_binary, engine in variants: + run = {"variant": name, "input_bytes": info["bytes"], "status": "ok", "warmups": [], "samples": []} + for _ in range(args.warmups): + sample = invoke(helper, target_binary, engine, path, info["arguments"], args.timeout) + run["warmups"].append(sample) + if sample["status"] != "ok": + run["status"] = "failed" + break + if run["status"] == "ok": + for _ in range(args.samples): + sample = invoke(helper, target_binary, engine, path, info["arguments"], args.timeout) + run["samples"].append(sample) + if sample["status"] != "ok": + run["status"] = "failed" + break + if run["status"] == "ok": + run["summary"] = summary(run["samples"]) + metrics = run["summary"] + print(f"{name}: {info['bytes']} bytes, " + f"{metrics['parser_seconds']['mean'] * 1000:.3f} +/- " + f"{metrics['parser_seconds']['sample_stdev'] * 1000:.3f} ms, " + f"{metrics['peak_rss_kib']['mean']:.0f} KiB mean peak RSS", file=sys.stderr) + else: + failed = True + print(f"FAILED {name} for {info['bytes']} bytes: inspect JSON report diagnostics", file=sys.stderr) + report["runs"].append(run) + + for label, artifact in artifacts.items(): + if file_hash(Path(artifact["path"])) != artifact["sha256"]: + failed = True + report.setdefault("errors", []).append(f"{label} changed during measurement; results are invalid") + report["status"] = "failed" if failed else "ok" + output = json.dumps(report, indent=2) + "\n" + if args.output: + args.output.write_text(output) + print(f"JSON report: {args.output}", file=sys.stderr) + else: + sys.stdout.write(output) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/benchmark_rusage.c b/tests/benchmark_rusage.c new file mode 100644 index 0000000..bf42813 --- /dev/null +++ b/tests/benchmark_rusage.c @@ -0,0 +1,47 @@ +#define _DEFAULT_SOURCE + +#include +#include +#include +#include +#include +#include + +/* Measure a target forked by a small, freshly exec'd process. Directly using + * Python's child rusage can include the Python parent's resident-memory floor. + * Linux reports ru_maxrss in KiB; benchmark_jsoncons.py checks the platform. + */ +int main(int argc, char **argv) { + if (argc < 2) { + fprintf(stderr, "Usage: benchmark_rusage EXECUTABLE [ARGUMENTS...]\n"); + return 2; + } + + pid_t child = fork(); + if (child < 0) { + perror("fork"); + return 2; + } + if (child == 0) { + execv(argv[1], argv + 1); + perror("execv"); + _exit(127); + } + + int status; + struct rusage usage; + while (wait4(child, &status, 0, &usage) < 0) { + if (errno == EINTR) continue; + perror("wait4"); + return 2; + } + double user = (double)usage.ru_utime.tv_sec + usage.ru_utime.tv_usec / 1000000.0; + double system = (double)usage.ru_stime.tv_sec + usage.ru_stime.tv_usec / 1000000.0; + fprintf(stderr, + "JSONBENCH_RUSAGE {\"peak_rss_kib\":%ld,\"user_seconds\":%.9f," + "\"system_seconds\":%.9f}\n", + usage.ru_maxrss, user, system); + + if (WIFEXITED(status)) return WEXITSTATUS(status); + return WIFSIGNALED(status) ? 128 + WTERMSIG(status) : 2; +} diff --git a/tests/jsoncons_api_tests.cpp b/tests/jsoncons_api_tests.cpp new file mode 100644 index 0000000..e474f1e --- /dev/null +++ b/tests/jsoncons_api_tests.cpp @@ -0,0 +1,316 @@ +// Direct regression checks for the C interface, including parser reuse and +// limits that cannot be represented by the CLI's positive-integer options. +#include "../src/jsonconsparser.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if HAVE_JSONCONS +namespace { + +class TemporaryInput { + std::string path_; + + public: + explicit TemporaryInput(const std::string& text) { + char name[] = "/tmp/jsonbench-jsoncons-api-XXXXXX"; + const int fd = mkstemp(name); + if (fd < 0) throw std::runtime_error("mkstemp failed"); + path_ = name; + std::size_t written = 0; + while (written < text.size()) { + const ssize_t count = write(fd, text.data() + written, text.size() - written); + if (count < 0 && errno == EINTR) continue; + if (count <= 0) { + close(fd); + unlink(path_.c_str()); + throw std::runtime_error("temporary input write failed"); + } + written += static_cast(count); + } + if (close(fd) != 0) { + unlink(path_.c_str()); + throw std::runtime_error("temporary input close failed"); + } + } + ~TemporaryInput() { unlink(path_.c_str()); } + TemporaryInput(const TemporaryInput&) = delete; + TemporaryInput& operator=(const TemporaryInput&) = delete; + const char* path() const { return path_.c_str(); } +}; + +class CapturedOutput { + FILE* file_ = nullptr; + int saved_fd_ = -1; + + public: + CapturedOutput() { + if (std::fflush(stdout) != 0) throw std::runtime_error("stdout flush failed"); + file_ = std::tmpfile(); + if (file_ == nullptr) throw std::runtime_error("stdout temporary file failed"); + saved_fd_ = dup(STDOUT_FILENO); + if (saved_fd_ < 0 || dup2(fileno(file_), STDOUT_FILENO) < 0) { + if (saved_fd_ >= 0) close(saved_fd_); + std::fclose(file_); + throw std::runtime_error("stdout capture failed"); + } + } + ~CapturedOutput() { + std::fflush(stdout); + dup2(saved_fd_, STDOUT_FILENO); + close(saved_fd_); + std::fclose(file_); + } + CapturedOutput(const CapturedOutput&) = delete; + CapturedOutput& operator=(const CapturedOutput&) = delete; + std::string read() { + if (std::fflush(stdout) != 0 || std::fseek(file_, 0, SEEK_SET) != 0) + throw std::runtime_error("captured output rewind failed"); + std::string text; + char buffer[1024]; + std::size_t count; + while ((count = std::fread(buffer, 1, sizeof buffer, file_)) != 0) + text.append(buffer, count); + if (std::ferror(file_)) throw std::runtime_error("captured output read failed"); + return text; + } +}; + +// Start each error slot with a caller-owned allocation. The API must reset +// the pointer, not free the caller's old value. Free returned diagnostics with +// free(), as required by the C interface, on success and failure paths alike. +class ErrorSlot { + char* original_; + + public: + char* value; + ErrorSlot() : original_(static_cast(std::malloc(1))), value(original_) { + if (original_ == nullptr) throw std::bad_alloc(); + *original_ = '\0'; + } + ~ErrorSlot() { + if (value != original_) std::free(value); + std::free(original_); + } + ErrorSlot(const ErrorSlot&) = delete; + ErrorSlot& operator=(const ErrorSlot&) = delete; + bool replaced() const { return value != original_; } +}; + +class Parser { + public: + jsoncons_parser* value = nullptr; + Parser() { + ErrorSlot error; + const int rc = jsoncons_json_init(&value, &error.value); + if (rc != 0 || value == nullptr || error.value != nullptr) { + jsoncons_json_cleanup(value); + throw std::runtime_error("parser initialization failed"); + } + } + ~Parser() { jsoncons_json_cleanup(value); } + Parser(const Parser&) = delete; + Parser& operator=(const Parser&) = delete; +}; + +struct ParseResult { + int status; + bool error_replaced; + bool error_is_null; + std::string error; + std::string output; +}; + +ParseResult parse_path(jsoncons_parser* parser, const char* path) { + ErrorSlot error; + CapturedOutput output; + const int status = jsoncons_parse_file(parser, path, &error.value); + return {status, error.replaced(), error.value == nullptr, + error.value != nullptr ? error.value : "", output.read()}; +} + +ParseResult parse(jsoncons_parser* parser, const std::string& text) { + TemporaryInput input(text); + return parse_path(parser, input.path()); +} + +class Checks { + unsigned int total_ = 0; + unsigned int failures_ = 0; + + public: + void check(const std::string& name, bool passed) { + ++total_; + if (!passed) { + ++failures_; + std::fprintf(stderr, "FAIL: %s\n", name.c_str()); + } + } + void success(const std::string& name, const ParseResult& result, + const std::string& expected = "") { + check(name + ": succeeds", result.status == 0); + check(name + ": clears previous error", result.error_replaced && result.error_is_null); + check(name + ": exact output", result.output == expected); + if (result.status != 0) + std::fprintf(stderr, " diagnostic: %s\n", result.error.c_str()); + } + void failure(const std::string& name, const ParseResult& result) { + check(name + ": fails", result.status != 0); + check(name + ": returns fresh malloc-owned diagnostic", + result.error_replaced && !result.error_is_null && !result.error.empty()); + } + int finish() const { + std::printf("jsoncons C API: %u checks passed, %u failed\n", + total_ - failures_, failures_); + return failures_ == 0 ? 0 : 1; + } +}; + +void invalid_arguments(Checks& checks) { + { + ErrorSlot error; + checks.check("init rejects null parser slot", jsoncons_json_init(nullptr, &error.value) != 0); + checks.check("failed init clears previous error", error.replaced() && error.value == nullptr); + } + jsoncons_parser* null_parser = nullptr; + checks.check("init rejects null error slot", jsoncons_json_init(&null_parser, nullptr) != 0); + checks.check("failed init leaves null parser", null_parser == nullptr); + checks.check("init rejects both null slots", jsoncons_json_init(nullptr, nullptr) != 0); + checks.check("cleanup accepts null", jsoncons_json_cleanup(nullptr) == 0); + checks.check("depth setter rejects null parser", jsoncons_set_max_depth(nullptr, 1) != 0); + checks.check("argument setter rejects null parser", jsoncons_set_max_arg_num(nullptr, 1) != 0); + checks.check("silence setter rejects null parser", jsoncons_set_silence(nullptr, 1) != 0); + + Parser parser; + TemporaryInput input("null"); + const ParseResult no_parser = parse_path(nullptr, input.path()); + checks.check("parse rejects null parser", no_parser.status != 0); + checks.check("invalid parser clears previous error", no_parser.error_replaced && no_parser.error_is_null); + const ParseResult no_filename = parse_path(parser.value, nullptr); + checks.check("parse rejects null filename", no_filename.status != 0); + checks.check("invalid filename clears previous error", no_filename.error_replaced && no_filename.error_is_null); + checks.check("parse rejects null error slot", + jsoncons_parse_file(parser.value, input.path(), nullptr) != 0); + checks.failure("missing file", parse_path(parser.value, (std::string(input.path()) + ".missing").c_str())); + checks.success("parse remains usable after invalid calls", parse(parser.value, "null"), ": \n"); +} + +void limit_setters(Checks& checks) { + Parser parser; + checks.check("set depth to two", jsoncons_set_max_depth(parser.value, 2) == 0); + checks.check("set arguments to two", jsoncons_set_max_arg_num(parser.value, 2) == 0); + checks.check("enable silence", jsoncons_set_silence(parser.value, 1) == 0); + const double upper = static_cast(UINT_MAX); + const std::vector> invalid = { + {"NaN", std::numeric_limits::quiet_NaN()}, + {"positive infinity", std::numeric_limits::infinity()}, + {"negative infinity", -std::numeric_limits::infinity()}, + {"negative integer", -1.0}, {"negative fraction", -0.5}, + {"fraction below one", 0.5}, {"positive fraction", 1.5}, + {"above UINT_MAX", upper + 1.0}, {"DBL_MAX", std::numeric_limits::max()} + }; + for (const auto& value : invalid) { + checks.check("depth rejects " + value.first, + jsoncons_set_max_depth(parser.value, value.second) != 0); + checks.check("argument count rejects " + value.first, + jsoncons_set_max_arg_num(parser.value, value.second) != 0); + checks.success("rejected " + value.first + " preserves limits", + parse(parser.value, "{\"a\":{\"b\":null,\"c\":true}}")); + checks.failure("rejected " + value.first + " preserves depth ceiling", + parse(parser.value, "[[[null]]]")); + checks.failure("rejected " + value.first + " preserves count ceiling", + parse(parser.value, "[null,true,false]")); + } + checks.check("depth accepts UINT_MAX", jsoncons_set_max_depth(parser.value, upper) == 0); + checks.check("argument count accepts UINT_MAX", jsoncons_set_max_arg_num(parser.value, upper) == 0); + checks.success("maximum representable limits permit ordinary JSON", + parse(parser.value, "{\"a\":[null,true,false]}")); +} + +void zero_limits(Checks& checks) { + Parser parser; + checks.check("set depth to zero", jsoncons_set_max_depth(parser.value, 0) == 0); + checks.check("enable silence for zero depth", jsoncons_set_silence(parser.value, 1) == 0); + for (const auto& text : {"null", "true", "false", "1", "\"value\""}) + checks.success(std::string("zero depth allows root ") + text, parse(parser.value, text)); + checks.failure("zero depth rejects object", parse(parser.value, "{}")); + checks.failure("zero depth rejects array", parse(parser.value, "[]")); + checks.check("restore depth for zero arguments", jsoncons_set_max_depth(parser.value, 3) == 0); + checks.check("set arguments to zero", jsoncons_set_max_arg_num(parser.value, 0) == 0); + for (const auto& text : {"{}", "[]", "{\"a\":[{},[]]}"}) + checks.success(std::string("zero arguments allows ") + text, parse(parser.value, text)); + for (const auto& text : {"null", "true", "false", "0", "\"\"", "{\"a\":null}", "[false]"}) + checks.failure(std::string("zero arguments rejects ") + text, parse(parser.value, text)); + checks.check("negative zero is an integral zero limit", jsoncons_set_max_depth(parser.value, -0.0) == 0); + checks.failure("negative-zero depth has zero semantics", parse(parser.value, "{}")); +} + +void boundaries_and_reuse(Checks& checks) { + Parser parser; + checks.check("set exact depth boundary", jsoncons_set_max_depth(parser.value, 3) == 0); + checks.check("set exact count boundary", jsoncons_set_max_arg_num(parser.value, 2) == 0); + checks.check("silence accepts any nonzero value", jsoncons_set_silence(parser.value, -7) == 0); + checks.success("array exactly at depth", parse(parser.value, "[[[null]]]")); + checks.failure("array above depth", parse(parser.value, "[[[[null]]]]")); + checks.success("object exactly at depth", parse(parser.value, "{\"a\":{\"b\":{\"c\":true}}}")); + checks.failure("object above depth", parse(parser.value, "{\"a\":{\"b\":{\"c\":{}}}}")); + checks.success("null and bool exactly at count", parse(parser.value, "{\"first\":null,\"second\":true}")); + checks.failure("null and bool above count", parse(parser.value, "{\"first\":null,\"second\":true,\"third\":false}")); + checks.success("empty containers consume no arguments", parse(parser.value, "{\"a\":{},\"b\":[],\"c\":null,\"d\":false}")); + checks.success("success resets count and path", parse(parser.value, "{\"new\":null,\"fresh\":false}")); + const ParseResult malformed = parse(parser.value, "{\"leaked\":{\"a\":true,\"b\":"); + checks.failure("nested truncated input", malformed); + checks.check("syntax error reports line", malformed.error.find("line ") != std::string::npos); + checks.check("syntax error reports column", malformed.error.find("column ") != std::string::npos); + checks.check("silent error produces no arguments", malformed.output.empty()); + checks.success("syntax failure resets count and path", parse(parser.value, "{\"fresh\":true}")); + checks.check("disable silence after reused parser", jsoncons_set_silence(parser.value, 0) == 0); + checks.success("output has fresh path after nested failure", parse(parser.value, "{\"fresh\":true}"), "fresh: true\n"); + checks.success("successful root array resets afterward", parse(parser.value, "[null,false]"), "array.array: \narray.array: false\n"); + checks.success("root scalar has fresh empty path", parse(parser.value, "2"), ": 2\n"); + const ParseResult count_error = parse(parser.value, "{\"a\":null,\"b\":true,\"c\":false}"); + checks.failure("retains count limit when unsilenced", count_error); + checks.check("count limit prints exactly permitted prefix", count_error.output == "a: \nb: true\n"); + checks.success("count failure resets parser", parse(parser.value, "{\"after\":false}"), "after: false\n"); + checks.check("tighten depth retained across calls", jsoncons_set_max_depth(parser.value, 1) == 0); + checks.failure("retains depth limit across calls", parse(parser.value, "{\"too\":[]}")); + checks.success("depth failure resets parser", parse(parser.value, "{\"after\":null}"), "after: \n"); + checks.check("reenable silence", jsoncons_set_silence(parser.value, 2) == 0); + checks.success("silence persists on successful reuse", parse(parser.value, "{\"quiet\":false}")); + checks.failure("silence still validates malformed JSON", parse(parser.value, "{\"quiet\":true,}")); + checks.success("silence persists after failed reuse", parse(parser.value, "{\"quiet\":true}")); +} + +} // namespace + +int main() { + try { + Checks checks; + invalid_arguments(checks); + limit_setters(checks); + zero_limits(checks); + boundaries_and_reuse(checks); + return checks.finish(); + } catch (const std::exception& error) { + std::fprintf(stderr, "FAIL: C API test setup: %s\n", error.what()); + return 1; + } +} +#else +int main() { + std::fprintf(stderr, "jsoncons C API tests require an enabled JSONCONS engine\n"); + return 77; +} +#endif diff --git a/tests/regression_tests.py b/tests/regression_tests.py new file mode 100755 index 0000000..303a960 --- /dev/null +++ b/tests/regression_tests.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python3 +"""Deterministic CLI regression checks, without an external test framework.""" + +import difflib +import os +from pathlib import Path +import re +import resource +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from typing import Optional + + +TESTS_DIR = Path(__file__).resolve().parent +# Remove only the final CLI timing trailer. Removing arbitrary Time: lines or +# blank lines would lose real argument data. Byte comparisons retain NUL/UTF-8. +TIMING_TEXT = rb"\nTime: (?:[0-9]+\.[0-9]+ sec|mean=[0-9]+\.[0-9]+ sec stddev=[0-9]+\.[0-9]+ sec \(N=[0-9]+\))\n\n" +TIMING_SUFFIX = re.compile(TIMING_TEXT + rb"\Z") +TIMING_PREFIX = re.compile(rb"\A" + TIMING_TEXT) + + +@dataclass +class Case: + name: str + document: bytes + expected: bytes + count: int + yajl_observed: Optional[bytes] = None + numeric_difference: bool = False + + def output_for(self, engine): + # YAJL preserves -0 and exponent '+'. jsoncons' public lossless API + # normalizes those spellings; other engines still must preserve value. + if self.numeric_difference and engine != "JSONCONS": + return self.yajl_observed + return self.expected + + +def utf8(text): + return text.encode("utf-8") + + +VALID_CASES = [ + Case("empty-object", b"{}", b"", 0), + Case("empty-array", b"[]", b"", 0), + Case("empty-containers", b'{"a":{},"b":[],"c":[{},[]]}', b"", 0), + Case("scalars", b'{"s":"text","t":true,"f":false,"n":null,"i":-42,"d":1.25}', + b"s: text\nt: true\nf: false\nn: \ni: -42\nd: 1.25\n", 6), + Case("root-string", b'"hello"', b": hello\n", 1), + Case("root-integer", b"42\n", b": 42\n", 1), + Case("root-integer-eof", b"42", b": 42\n", 1, b""), + Case("root-decimal", b"1.25\n", b": 1.25\n", 1), + Case("root-decimal-eof", b"1.25", b": 1.25\n", 1, b""), + Case("root-true", b"true", b": true\n", 1), + Case("root-false", b"false", b": false\n", 1), + Case("root-null", b"null", b": \n", 1), + Case("root-array", b'[1,true,null,"s"]', + b"array.array: 1\narray.array: true\narray.array: \narray.array: s\n", 4), + Case("nested-arrays", b"[[0,1],[2,3]]", + b"array.array.array: 0\narray.array.array: 1\narray.array.array: 2\narray.array.array: 3\n", 4), + Case("nested-objects", b'{"a":{"b":{"c":1}},"after":2}', b"a.b.c: 1\nafter: 2\n", 2), + Case("mixed-arrays", b'{"a":[1,[2,3],4],"b":{"c":5},"end":6}', + b"a.a: 1\na.a.a: 2\na.a.a: 3\na.a: 4\nb.c: 5\nend: 6\n", 6), + Case("array-objects", b'{"rows":[{"a":1,"b":2},{"a":3}],"after":4}', + b"rows.rows.a: 1\nrows.rows.b: 2\nrows.rows.a: 3\nafter: 4\n", 4), + Case("array-parent-key", b'[{"a":1},2,{"b":3},4]', + b"array.array.a: 1\narray.array: 2\narray.array.b: 3\narray.array: 4\n", 4), + Case("array-after-empty", b'{"a":[{"x":[]},1,{},2,[],3]}', + b"a.a: 1\na.a: 2\na.a: 3\n", 3), + Case("repeated-keys", b'{"k":1,"k":2,"nest":{"k":3,"k":4}}', + b"k: 1\nk: 2\nnest.k: 3\nnest.k: 4\n", 4), + Case("empty-keys", b'{"":{"":1},"last":2}', b": 1\nlast: 2\n", 2), + Case("dotted-keys", b'{"a.b":{"c":1},"d":2}', + b"a.b.c: 1\nd: 2\n", 2, b"a.b.c: 1\na.d: 2\n"), + Case("unicode", utf8('{"Grüße":"日本語 😀","escaped":"\\u20ac\\ud83d\\ude00"}'), + utf8("Grüße: 日本語 😀\nescaped: €😀\n"), 2), + Case("literal-unicode-escape", b'{"literal":"\\\\udc00"}', b"literal: \\udc00\n", 1), + Case("surrogate-across-buffer", b" " * 16379 + b'"\\ud83d\\ude00"', utf8(": 😀\n"), 1), + Case("escaped-backslash-across-buffer", b" " * 16382 + b'"\\\\udc00"', b": \\udc00\n", 1), + Case("utf8-across-buffer", b" " * 16381 + utf8('"😀"'), utf8(": 😀\n"), 1), + Case("escaped-strings", b'{"s":"quote:\\" slash:\\/ backslash:\\\\ tab:\\t line:\\n end"}', + b's: quote:" slash:/ backslash:\\ tab:\t line:\n end\n', 1), + Case("embedded-nul", b'{"a\\u0000b":"x\\u0000y"}', b"a\0b: x\0y\n", 1, b"a: x\n"), + Case("timing-like-value", b'{"Time":"0.000000000 sec","blank":"\\n"}', + b"Time: 0.000000000 sec\nblank: \n\n", 2), + Case("integer-boundaries", + b'{"min":-9223372036854775808,"max":9223372036854775807,"u":18446744073709551615,"beyond":18446744073709551616,"negative":-9223372036854775809}', + b"min: -9223372036854775808\nmax: 9223372036854775807\nu: 18446744073709551615\nbeyond: 18446744073709551616\nnegative: -9223372036854775809\n", 5), + Case("negative-zero", b'{"zero":0,"negative":-0,"decimal":-0.0}', + b"zero: 0\nnegative: 0\ndecimal: -0.0\n", 3, + b"zero: 0\nnegative: -0\ndecimal: -0.0\n", True), + Case("decimal-precision", + b'{"decimal":1.2300,"precise":0.123456789012345678901234567890,"tiny":1e-10000,"huge":1E9999}', + b"decimal: 1.2300\nprecise: 0.123456789012345678901234567890\ntiny: 1e-10000\nhuge: 1E9999\n", 4), + Case("exponents", b'{"plus":1e+03,"capital":1E-03,"zeros":1e-0003,"huge":1E+9999}', + b"plus: 1e03\ncapital: 1E-03\nzeros: 1e-0003\nhuge: 1E9999\n", 4, + b"plus: 1e+03\ncapital: 1E-03\nzeros: 1e-0003\nhuge: 1E+9999\n", True), +] + +INVALID_CASES = [ + ("empty-input", b""), ("whitespace-only", b" \t\r\n"), + ("truncated-object", b'{"a":1'), ("truncated-array", b"[1,2"), + ("truncated-string", b'{"s":"text'), ("truncated-literal", b"[tru"), + ("missing-closing-brace", b'{"a":{"b":1}'), ("missing-closing-bracket", b'{"a":[1,2}'), + ("missing-colon", b'{"a" 1}'), ("missing-comma", b"[1 2]"), ("wrong-separator", b'{"a"=1}'), + ("trailing-array-comma", b"[1,]"), ("trailing-object-comma", b'{"a":1,}'), + ("invalid-escape", b'"\\q"'), ("short-unicode-escape", b'"\\u123"'), + ("unpaired-high-surrogate", b'"\\ud800"'), ("unpaired-low-surrogate", b'"\\udc00"'), + ("high-surrogate-with-nonsurrogate", b'"\\ud800\\u0041"'), + ("high-surrogate-with-high-surrogate", b'"\\ud800\\ud800"'), + ("low-surrogate-across-buffer", b" " * 16379 + b'"\\udc00"'), + ("invalid-surrogate-pair-across-buffer", b" " * 16377 + b'"\\ud800\\u0041"'), + ("unescaped-newline", b'"a\nb"'), ("utf8-continuation", b'"\x80"'), + ("utf8-overlong", b'"\xc0\xaf"'), ("utf8-truncated", b'"\xe2\x82"'), + ("utf8-surrogate", b'"\xed\xa0\x80"'), ("utf8-out-of-range", b'"\xf4\x90\x80\x80"'), + ("utf8-invalid-key", b'{"\xff":1}'), ("extra-document", b"{} []"), + ("extra-text", b'{"a":1}xyz'), ("extra-nul", b"{}\0"), + ("number-leading-zero", b"[01]"), ("number-leading-plus", b"[+1]"), + ("number-missing-integer", b"[.1]"), ("number-missing-fraction", b"[1.]"), + ("number-missing-exponent", b"[1e]"), ("number-missing-exponent-value", b"[1e+]"), + ("number-double-sign", b"[--1]"), ("number-nan", b"[NaN]"), + ("number-infinity", b"[Infinity]"), ("comment", b"{/* comment */}"), +] + + +def disable_core_dumps(): + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + + +class Suite: + def __init__(self, binary, selected, available, temp): + self.binary, self.selected, self.available, self.temp = binary, selected, available, Path(temp) + self.passed = self.failed = self.skipped = self.differences = 0 + + def report(self, status, label): + print(f"{status:<5} {label}", flush=True) + field = {"PASS": "passed", "FAIL": "failed", "SKIP": "skipped", "DIFF": "differences"}[status] + setattr(self, field, getattr(self, field) + 1) + + def fixture(self, name, document): + path = self.temp / f"{name}.json" + path.write_bytes(document) + return path + + def run(self, engine, path, options=(), stdout=subprocess.PIPE): + command = [str(self.binary), "-e", engine, *map(str, options), str(path)] + try: + return subprocess.run(command, stdout=stdout, stderr=subprocess.PIPE, timeout=15, + preexec_fn=disable_core_dumps, check=False) + except subprocess.TimeoutExpired as error: + return subprocess.CompletedProcess(command, 124, error.stdout or b"", (error.stderr or b"") + b"\nTest timed out\n") + + @staticmethod + def arguments(result): + iterations = 1 + for index, arg in enumerate(result.args[:-1]): + if arg == "-n": + iterations = int(result.args[index + 1]) + # For -n>1 the CLI prints timing before the final output pass. + if iterations > 1: + match = TIMING_PREFIX.search(result.stdout) + return result.stdout[match.end():] if match else None + match = TIMING_SUFFIX.search(result.stdout) + return result.stdout[:match.start()] if match else None + + @staticmethod + def diagnostic(result, expected=None, actual=None): + print(f" exit status: {result.returncode}; command: {' '.join(result.args)}") + if result.stderr: + print(" stderr: " + repr(result.stderr[:1200])) + if expected is not None and actual is not None: + print(f" expected {len(expected)} output bytes, got {len(actual)}") + lines = difflib.unified_diff(expected.decode("utf-8", "backslashreplace").splitlines(True), + actual.decode("utf-8", "backslashreplace").splitlines(True), + fromfile="expected", tofile="actual") + for index, line in enumerate(lines): + if index == 12: + print(" ... diff truncated ...") + break + print(" " + repr(line.rstrip("\n")[:300])) + elif result.stdout: + print(" stdout: " + repr(result.stdout[:1200])) + + def success(self, engine, name, path, expected=None, options=(), silent=False): + label = f"[{engine}] {name}" + options = ["-a", "9999", *options, *(["-s"] if silent else [])] + result = self.run(engine, path, options) + actual = self.arguments(result) + if result.returncode != 0 or actual is None: + self.report("FAIL", label + (" (missing timing/output trailer)" if actual is None else "")) + self.diagnostic(result) + return None + if silent: + expected = b"" + if expected is not None and actual != expected: + self.report("FAIL", label + " (argument output mismatch)") + self.diagnostic(result, expected, actual) + return None + self.report("PASS", label) + return actual + + def rejected(self, engine, name, path, options=(), diagnostic=None, silent=True): + result = self.run(engine, path, ["-a", "9999", *options, *(["-s"] if silent else [])]) + self.rejected_result(engine, name, result, diagnostic) + + def rejected_result(self, engine, name, result, diagnostic=None): + # Signals/timeouts are failures, not successful validation. Unsupported + # warnings alone do not establish that malformed input was rejected. + errors = [line for line in result.stderr.splitlines() if b"does not support" not in line] + good = 0 < result.returncode < 124 and bool(b"\n".join(errors).strip()) + if diagnostic is not None: + good = good and diagnostic.lower() in result.stderr.lower() + if good: + self.report("PASS", f"[{engine}] {name}") + else: + self.report("FAIL", f"[{engine}] {name} (expected rejection with a diagnostic)") + self.diagnostic(result) + + def limit_rejected(self, engine, name, path, options, diagnostic=None, silent=True): + result = self.run(engine, path, ["-a", "9999", *options, *(["-s"] if silent else [])]) + warning = b"does not support max depth limit" if "depth" in name else b"does not support max arg num limit" + if 0 <= result.returncode < 124 and warning in result.stderr: + self.report("SKIP", f"[{engine}] {name} (adapter reports unsupported limit; no enforcement claim)") + else: + self.rejected_result(engine, name, result, diagnostic) + + def baseline(self): + fixtures = ["test01.json", "test02.json", "test03.json", "test04.json", "test06.json", "512KB.json"] + reference = {} + if "YAJL" in self.available: + for fixture in fixtures: + result = self.run("YAJL", TESTS_DIR / fixture, ["-a", "9999"]) + output = self.arguments(result) + if result.returncode == 0 and output: + reference[fixture] = output + else: + self.report("FAIL", f"[YAJL] baseline reference: {fixture}") + self.diagnostic(result) + for engine in self.selected: + for fixture in fixtures: + path = TESTS_DIR / fixture + self.success(engine, f"existing valid parse/silent: {fixture}", path, silent=True) + if fixture in reference: + self.success(engine, f"existing argument output vs YAJL: {fixture}", path, reference[fixture]) + else: + self.report("SKIP", f"[{engine}] output comparison: {fixture} (YAJL unavailable; compatibility untested)") + self.rejected(engine, "reject existing invalid fixture: test05.json", TESTS_DIR / "test05.json") + self.limit_rejected(engine, "existing depth limit (-d 1)", TESTS_DIR / "test06.json", ["-d", "1"]) + self.limit_rejected(engine, "existing argument limit (-a 1)", TESTS_DIR / "test06.json", ["-a", "1"]) + + def matrix(self): + for engine in self.selected: + for case in VALID_CASES: + path = self.fixture(case.name, case.document) + self.success(engine, f"golden arguments ({case.count}): {case.name}", path, + case.output_for(engine), ["-a", str(max(1, case.count))]) + self.success(engine, f"silent validation: {case.name}", path, + options=["-a", str(max(1, case.count))], silent=True) + if case.count > 1: + self.limit_rejected(engine, f"argument count exceeds {case.count - 1}: {case.name}", + path, ["-a", str(case.count - 1)]) + for name, document in INVALID_CASES: + path = self.fixture(name, document) + self.rejected(engine, f"invalid JSON (silent): {name}", path) + self.rejected(engine, f"invalid JSON (output): {name}", path, silent=False) + self.boundaries(engine) + self.large_document(engine) + self.operational(engine) + self.compatibility() + + def compatibility(self): + if "JSONCONS" not in self.selected: + return + for case in VALID_CASES: + if "YAJL" not in self.available: + self.report("SKIP", f"[JSONCONS/YAJL] {case.name} (YAJL unavailable; compatibility untested)") + continue + path = self.fixture(case.name, case.document) + jsoncons = self.run("JSONCONS", path, ["-a", "9999"]) + yajl = self.run("YAJL", path, ["-a", "9999"]) + jsoncons_output, yajl_output = self.arguments(jsoncons), self.arguments(yajl) + observed = case.expected if case.yajl_observed is None else case.yajl_observed + if jsoncons.returncode != 0 or jsoncons_output != case.expected or yajl.returncode != 0 or yajl_output != observed: + self.report("FAIL", f"[JSONCONS/YAJL] comparison prerequisite: {case.name}") + self.diagnostic(jsoncons, case.expected, jsoncons_output) + self.diagnostic(yajl, observed, yajl_output) + elif jsoncons_output == yajl_output: + self.report("PASS", f"[JSONCONS/YAJL] byte-exact compatibility: {case.name}") + else: + self.report("DIFF", f"[JSONCONS/YAJL] {case.name} (both outputs checked; documented difference, no compatibility claim)") + + def config_value(self, name, fallback): + config = self.binary.parent.parent / "config.h" + if not config.is_file(): + config = TESTS_DIR.parent / "config.h" + if config.is_file(): + found = re.search(r"^#define\s+" + name + r"\s+([0-9]+)", config.read_text(), re.MULTILINE) + if found: + return int(found.group(1)) + return fallback + + def boundaries(self, engine): + depth = 64 + for kind, document, expected in [ + ("arrays", b"[" * depth + b"1" + b"]" * depth, + b".".join([b"array"] * (depth + 1)) + b": 1\n"), + ("objects", b'{"k":' * depth + b"1" + b"}" * depth, + b".".join([b"k"] * depth) + b": 1\n")]: + path = self.fixture("deep-" + kind, document) + self.success(engine, f"{kind} exactly at depth 64", path, expected, ["-d", str(depth)]) + self.limit_rejected(engine, f"{kind} exceed depth 63", path, ["-d", str(depth - 1)]) + default_depth = self.config_value("LIMIT_DEPTH", 500) + exact = self.fixture("default-depth", b"[" * default_depth + b"0" + b"]" * default_depth) + self.success(engine, f"default depth accepts {default_depth} containers", exact, + b".".join([b"array"] * (default_depth + 1)) + b": 0\n") + path = self.fixture("excessive-depth", b"[" * (default_depth + 1) + b"0" + b"]" * (default_depth + 1)) + self.limit_rejected(engine, f"default depth rejects {default_depth + 1} containers", path, []) + for root in [b"{}", b"[]", b"1\n"]: + self.success(engine, f"root at depth 1: {root!r}", self.fixture("depth-root", root), + b": 1\n" if root.startswith(b"1") else b"", ["-d", "1"]) + for name, document in [("booleans", b"[true,false]"), ("nulls", b"[null,null]")]: + self.limit_rejected(engine, f"{name} enforce argument limit with output", self.fixture(name, document), + ["-a", "1"], silent=False) + size = self.config_value("JSON_STRING_SIZE", 8192) + key, value, child = b"k" * (size - 1), b"v" * (size - 1), b"b" * (size - 3) + length_cases = [ + ("key", b'{"' + key + b'":1}', key + b": 1\n", b'{"' + key + b'k":1}'), + ("value", b'{"k":"' + value + b'"}', b"k: " + value + b"\n", b'{"k":"' + value + b'v"}'), + ("combined-path", b'{"a":{"' + child + b'":1}}', b"a." + child + b": 1\n", b'{"a":{"' + child + b'b":1}}')] + for name, document, expected, oversized in length_cases: + for silence in (False, True): + mode = "silent" if silence else "output" + self.success(engine, f"{name} length {size - 1} ({mode})", self.fixture(name, document), expected, silent=silence) + self.rejected(engine, f"{name} length {size} rejected ({mode})", self.fixture(name + "-long", oversized), silent=silence) + giant = b"x" * (2 * 1024 * 1024) + for name, document in [("huge-string", b'{"k":"' + giant + b'"}'), + ("huge-number", b'{"k":1' + b"0" * len(giant) + b"}"), + ("huge-unterminated-string", b'{"k":"' + giant)]: + self.rejected(engine, f"bounded hostile token (silent): {name}", self.fixture(name, document)) + self.rejected(engine, f"bounded hostile token (output): {name}", self.fixture(name, document), silent=False) + + def large_document(self, engine): + ceiling = self.config_value("FILE_BUFFER_SIZE", 10485760) + payload = b"x" * 512 + record = b'{"id":123,"pad":"' + payload + b'"}' + count = max(2000, (ceiling + 1024) // (len(record) + 1) + 1) + path = self.fixture("large-stream", b"[" + b",".join([record] * count) + b"]") + expected = (b"array.array.id: 123\narray.array.pad: " + payload + b"\n") * count + self.success(engine, f"large document ({path.stat().st_size} bytes, {count * 2} arguments)", path, + expected, ["-a", str(count * 2)]) + self.success(engine, "large document silent validation", path, options=["-a", str(count * 2)], silent=True) + self.limit_rejected(engine, "large document argument boundary", path, ["-a", str(count * 2 - 1)]) + + def operational(self, engine): + self.rejected(engine, "missing input file diagnostic", self.temp / "does-not-exist.json") + path = self.fixture("iterations", b'{"x":1,"x":2}') + self.success(engine, "three timed iterations print arguments once", path, b"x: 1\nx: 2\n", ["-n", "3", "-a", "2"]) + self.success(engine, "three silent iterations emit no arguments", path, options=["-n", "3", "-a", "2"], silent=True) + self.limit_rejected(engine, "timed iterations retain argument limit", path, ["-n", "3", "-a", "1"]) + for option in ("-d", "-a"): + for value in ("-1", "abc", "1x", "1.5", "4294967296", "0"): + self.rejected(engine, f"CLI rejects {option} {value}", path, [option, value]) + if Path("/dev/full").exists(): + for iterations in (1, 2): + with open("/dev/full", "wb") as output: + result = self.run(engine, path, ["-n", str(iterations)], stdout=output) + if 0 < result.returncode < 124 and result.stderr.strip(): + self.report("PASS", f"[{engine}] output error propagated (-n {iterations})") + else: + self.report("FAIL", f"[{engine}] output error ignored (-n {iterations})") + self.diagnostic(result) + else: + self.report("SKIP", f"[{engine}] output error test (/dev/full unavailable)") + + def timing(self): + try: + runs = int(os.environ.get("JSONBENCH_TIMING_RUNS", "0")) + except ValueError: + self.report("FAIL", "JSONBENCH_TIMING_RUNS must be a nonnegative integer") + return + if runs < 0: + self.report("FAIL", "JSONBENCH_TIMING_RUNS must be a nonnegative integer") + elif runs: + print(f"\n=== Timing: {runs} iterations per fixture, silent mode ===") + for fixture in ["test01.json", "test06.json", "512KB.json"]: + for engine in self.selected: + result = self.run(engine, TESTS_DIR / fixture, ["-s", "-a", "9999", "-n", str(runs)]) + if result.returncode != 0 or self.arguments(result) != b"": + self.report("FAIL", f"[{engine}] timing: {fixture}") + self.diagnostic(result) + else: + print(f"{engine:14} {fixture:12} {result.stdout.decode().strip()}") + + +def main(): + if len(sys.argv) not in (2, 3): + print("Usage: regression_tests.py BINARY [JSON_ENGINE]", file=sys.stderr) + return 1 + binary = Path(sys.argv[1]).resolve() + try: + result = subprocess.run([str(binary), "-h"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=15, preexec_fn=disable_core_dumps, check=False) + except (OSError, subprocess.TimeoutExpired) as error: + print(f"ERROR: cannot discover engines: {error}", file=sys.stderr) + return 1 + available = re.findall(r"^\s*-\s+([A-Z]+)\s*$", result.stdout.decode(), re.MULTILINE) + if result.returncode != 0 or not available: + print("ERROR: cannot discover engines from binary -h", file=sys.stderr) + return 1 + selected = available + if len(sys.argv) == 3: + if sys.argv[2] not in available: + print(f"ERROR: engine '{sys.argv[2]}' unavailable (available: {' '.join(available)})", file=sys.stderr) + return 1 + selected = [sys.argv[2]] + print(f"=== Available engines: {' '.join(available)} ===") + print(f"=== Selected engines: {' '.join(selected)} ===") + with tempfile.TemporaryDirectory(prefix="jsonbench-") as temp: + suite = Suite(binary, selected, available, temp) + suite.baseline() + suite.matrix() + suite.timing() + print(f"\n=== Results: {suite.passed} passed, {suite.failed} failed, {suite.skipped} skipped, " + f"{suite.differences} checked behavioral differences ===") + return 1 if suite.failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 316f940..3cb75ef 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -1,192 +1,22 @@ #!/usr/bin/env bash set -euo pipefail -BINARY="$(cd "$(dirname "$0")/.." && pwd)/src/jsonbench" TESTS_DIR="$(cd "$(dirname "$0")" && pwd)" -TMPDIR_LOCAL=$(mktemp -d "${TMPDIR:-/tmp}/jsonbench.XXXXXX") -trap 'rm -rf "$TMPDIR_LOCAL"' EXIT +BINARY="${JSONBENCH_BINARY:-$TESTS_DIR/../src/jsonbench}" if [[ ! -x "$BINARY" ]]; then - echo "ERROR: binary not found or not executable: $BINARY" - echo " Run './autogen.sh && ./configure && make' first" + echo "ERROR: binary not found or not executable: $BINARY" >&2 + echo " Run './autogen.sh && ./configure && make' first" >&2 exit 1 fi - -# Discover available engines from jsonbench -h output -ENGINES=() -while IFS= read -r line; do - if [[ "$line" =~ [[:space:]]-[[:space:]]([A-Z]+)$ ]]; then - ENGINES+=("${BASH_REMATCH[1]}") - fi -done < <("$BINARY" -h 2>&1 || true) - -if [[ ${#ENGINES[@]} -eq 0 ]]; then - echo "ERROR: no engines found in '$BINARY -h' output" +if ! command -v python3 >/dev/null 2>&1; then + echo "ERROR: Python 3 is required by the regression test helper" >&2 exit 1 fi - -# Optional: filter to a single engine -if [[ "${1:-}" != "" ]]; then - ENGINE_FILTER="$1" - FOUND=0 - for e in "${ENGINES[@]}"; do - [[ "$e" == "$ENGINE_FILTER" ]] && FOUND=1 - done - if [[ "$FOUND" -eq 0 ]]; then - echo "ERROR: engine '$ENGINE_FILTER' is not available (available: ${ENGINES[*]})" - exit 1 - fi - ENGINES=("$ENGINE_FILTER") +if [[ $# -gt 1 ]]; then + echo "Usage: $0 [JSON_ENGINE]" >&2 + exit 1 fi -PASS=0 -FAIL=0 -SKIP=0 - -# Valid JSON fixtures; test05.json is intentionally invalid and excluded here -VALID_FIXTURES=(test01.json test02.json test03.json test04.json test06.json 512KB.json) -INVALID_FIXTURES=(test05.json) - -# Use YAJL as the reference engine for output consistency; fall back to first available -REFERENCE_ENGINE="${ENGINES[0]}" -for e in "${ENGINES[@]}"; do - if [[ "$e" == "YAJL" ]]; then - REFERENCE_ENGINE="YAJL" - break - fi -done - -# Generate reference output for each valid fixture -echo "=== Reference engine: $REFERENCE_ENGINE ===" -for fixture in "${VALID_FIXTURES[@]}"; do - "$BINARY" -e "$REFERENCE_ENGINE" -a 9999 "$TESTS_DIR/$fixture" \ - > "$TMPDIR_LOCAL/ref_${fixture}.raw" 2>/dev/null - grep -Ev "^(Time:|$)" "$TMPDIR_LOCAL/ref_${fixture}.raw" \ - > "$TMPDIR_LOCAL/ref_${fixture}.out" || true -done -echo "" - -# --- Helpers --- - -pass() { echo "PASS $1"; PASS=$((PASS + 1)); } -fail() { echo "FAIL $1"; FAIL=$((FAIL + 1)); } -skip() { echo "SKIP $1"; SKIP=$((SKIP + 1)); } - -# Run a parse that must succeed (exit 0) -check_valid() { - local engine="$1" fixture="$2" - local label="[$engine] valid parse: $fixture" - local rc=0 - "$BINARY" -e "$engine" -s -a 9999 "$TESTS_DIR/$fixture" \ - > "$TMPDIR_LOCAL/out.txt" 2>&1 || rc=$? - if [[ "$rc" -eq 0 ]]; then pass "$label"; else fail "$label"; fi -} - -# Check that non-silent output matches the reference engine's output -check_consistency() { - local engine="$1" fixture="$2" - local label="[$engine] output consistency: $fixture" - local rc=0 - "$BINARY" -e "$engine" -a 9999 "$TESTS_DIR/$fixture" \ - > "$TMPDIR_LOCAL/cmp_${engine}_${fixture}.raw" 2>/dev/null || rc=$? - grep -Ev "^(Time:|$)" "$TMPDIR_LOCAL/cmp_${engine}_${fixture}.raw" \ - > "$TMPDIR_LOCAL/cmp_${engine}_${fixture}.out" || true - if [[ "$rc" -ne 0 ]]; then - fail "$label (parse failed)" - return - fi - if [[ ! -s "$TMPDIR_LOCAL/cmp_${engine}_${fixture}.out" ]]; then - skip "$label (engine produces no field output)" - return - fi - if diff -q "$TMPDIR_LOCAL/ref_${fixture}.out" \ - "$TMPDIR_LOCAL/cmp_${engine}_${fixture}.out" > /dev/null 2>&1; then - pass "$label" - else - fail "$label" - diff "$TMPDIR_LOCAL/ref_${fixture}.out" \ - "$TMPDIR_LOCAL/cmp_${engine}_${fixture}.out" | head -20 - fi -} - -# Run a parse that must fail (non-zero exit); handles "does not support" gracefully -# Usage: check_rejected engine label [extra args...] -check_rejected() { - local engine="$1" label="$2" - shift 2 - local rc=0 - "$BINARY" -e "$engine" -s "$@" \ - > "$TMPDIR_LOCAL/out.txt" 2>"$TMPDIR_LOCAL/err.txt" || rc=$? - if [[ "$rc" -ne 0 ]]; then - pass "[$engine] $label" - elif grep -q "does not support" "$TMPDIR_LOCAL/err.txt" 2>/dev/null; then - skip "[$engine] $label (engine does not support this limit)" - else - fail "[$engine] $label (expected non-zero exit, got 0)" - fi -} - -# --- Tests --- - -for engine in "${ENGINES[@]}"; do - # 1. Valid parse - for fixture in "${VALID_FIXTURES[@]}"; do - check_valid "$engine" "$fixture" - done - - # 2. Output consistency vs reference (skip self-comparison) - if [[ "$engine" != "$REFERENCE_ENGINE" ]]; then - for fixture in "${VALID_FIXTURES[@]}"; do - check_consistency "$engine" "$fixture" - done - fi - - # 3. Invalid JSON must be rejected - for fixture in "${INVALID_FIXTURES[@]}"; do - check_rejected "$engine" "rejects invalid JSON: $fixture" \ - "$TESTS_DIR/$fixture" - done - - # 4. Depth limit enforcement - check_rejected "$engine" "depth limit (-d 1) on test06.json" \ - -d 1 "$TESTS_DIR/test06.json" - - # 5. Arg limit enforcement - check_rejected "$engine" "arg limit (-a 1) on test06.json" \ - -a 1 "$TESTS_DIR/test06.json" -done - -# --- Summary --- -echo "" -echo "=== Results: $PASS passed, $FAIL failed, $SKIP skipped ===" - -# --- Timing table --- -BENCH_N=20 -echo "" -echo "=== Timing: mean of ${BENCH_N} runs (seconds, silent mode) ===" -echo "" - -printf "%-20s" "Fixture" -for engine in "${ENGINES[@]}"; do - printf "%12s" "$engine" -done -printf "\n" -printf "%-20s" "--------------------" -for engine in "${ENGINES[@]}"; do - printf "%12s" "------------" -done -printf "\n" - -for fixture in "${VALID_FIXTURES[@]}"; do - printf "%-20s" "$fixture" - for engine in "${ENGINES[@]}"; do - raw=$("$BINARY" -e "$engine" -s -a 9999 -n "$BENCH_N" \ - "$TESTS_DIR/$fixture" 2>/dev/null) || true - mean=$(printf '%s\n' "$raw" | grep "^Time:" | grep -oE "mean=[0-9.]+" | sed 's/mean=//') - printf "%12s" "${mean:-ERR}" - done - printf "\n" -done -echo "" - -[[ "$FAIL" -eq 0 ]] +# Standard-library helper, preserving the existing CLI and engine filter. +exec python3 "$TESTS_DIR/regression_tests.py" "$BINARY" "$@" From 10644d7fdf53face15d4b093962157e0f9ce8890 Mon Sep 17 00:00:00 2001 From: jens Steinbacher Date: Tue, 13 Oct 2026 13:42:28 +0000 Subject: [PATCH 02/18] Stream and finalize YAJL parsing and harden argument paths --- docs/adapter-stack.md | 73 ++++ docs/jsoncons-sax.md | 3 + docs/validation-results.md | 3 + src/jsonbench.c | 94 +++++- src/pathhandler.h | 172 ++++------ src/yajlparser.c | 642 +++++++++++++++--------------------- src/yajlparser.h | 16 + tests/pathhandler_tests.cpp | 217 ++++++++++++ tests/regression_tests.py | 8 +- tests/yajl_api_tests.cpp | 480 +++++++++++++++++++++++++++ 10 files changed, 1226 insertions(+), 482 deletions(-) create mode 100644 docs/adapter-stack.md create mode 100644 tests/pathhandler_tests.cpp create mode 100644 tests/yajl_api_tests.cpp diff --git a/docs/adapter-stack.md b/docs/adapter-stack.md new file mode 100644 index 0000000..9304890 --- /dev/null +++ b/docs/adapter-stack.md @@ -0,0 +1,73 @@ +# Parser follow-up stack + +These changes build on the jsoncons SAX/test PR +[#4](https://github.com/digitalwave/jsonbench/pull/4). The earlier +`validation-results.md` records that original snapshot; it is not the result +of these follow-ups. All engine assertions remain enabled. + +## 1. YAJL streaming and shared path safety + +YAJL now reads bounded 16 KiB chunks, finalizes with `yajl_complete_parse`, and +implements its declared incremental `process`/`complete` API. Root numbers at +physical EOF are emitted; truncated and extra documents fail. Parsing a new file +resets state while retaining settings. C entry points validate pointers and +clean up partial allocations. Containers count toward depth, every scalar +(including boolean/null) counts toward the argument limit, and silent mode +checks the same boundaries. + +Checked copies and structural frames replace unchecked key/prefix operations. +Decoded NULs are emitted by length. Narrow raw-UTF-8 and surrogate guards cover +YAJL 2.1's incomplete scalar validation, including overlong encodings, surrogate +codepoints and values above U+10FFFF. A bounded raw-token guard prevents the +lexer from accumulating arbitrarily large individual strings/numbers before +its callbacks. It does not replace native JSON grammar parsing. + +The C++ `PathHandler` similarly records structural boundaries instead of +searching for dots. Existing callback method names remain compatible. Keys, +values and full argument names must be shorter than `JSON_STRING_SIZE` (8192 +bytes by default). Allocation failures in legacy noexcept path callbacks become +an error flag. The CLI now propagates failed non-timed output passes and final +stdout errors for every engine. + +Four old YAJL comparison exceptions are removed: dotted-key restoration, +embedded NUL output, and the two root-number-at-EOF cases must now match their +explicit expected output. Numeric lexical differences for jsoncons (`-0` and +exponent `+`) remain separately checked. + +Fresh verification of this step: YAJL CLI matrix **232 passed, 0 failed, 0 +skipped**; direct C API **3900 checks passed**; shared path contract **181 checks +passed**. Both direct harnesses also passed ASan/UBSan with leak detection. + +```sh +source /workspace/.jsonbench-tools/activate.sh +./autogen.sh && ./configure && make -j2 +tests/run_tests.sh YAJL +# Standalone public API checks (use the configured include/link flags): +gcc -std=gnu11 -Wall -Wextra -Werror $CPPFLAGS -c src/yajlparser.c -o /tmp/yajl.o +g++ -std=c++17 -Wall -Wextra -Werror $CPPFLAGS \ + tests/yajl_api_tests.cpp /tmp/yajl.o $LDFLAGS -lyajl -o /tmp/yajl-api-tests +/tmp/yajl-api-tests +g++ -std=c++17 -Wall -Wextra -Werror $CPPFLAGS \ + tests/pathhandler_tests.cpp -o /tmp/pathhandler-tests +/tmp/pathhandler-tests +``` + +## Optional JSON Schema validation + +No schema validator dependency is introduced. Schema rules are a separate +concern from the streaming parser's JSON grammar, Unicode and resource checks. +The public APIs of both pboettch/json-schema-validator and Sourcemeta Blaze +validate a DOM, so they are not a drop-in SAX schema evaluator. An additional +schema pass would incur DOM memory and must stay outside parser benchmark +measurements. + +The [pboettch validator](https://github.com/pboettch/json-schema-validator) +supports Draft 7 and is MIT-licensed; it uses nlohmann's JSON DOM and does not +support arbitrary numeric ranges. [Blaze](https://github.com/sourcemeta/blaze) +supports newer dialects, including 2019-09 and 2020-12, and uses Sourcemeta's +JSON DOM. Its license is AGPLv3 (or a commercial alternative). jsonbench's +GPLv3 permits an AGPLv3 combination under section 13, with the additional AGPL +conditions. ModSecurity v3 is Apache-2.0 licensed: adding AGPL code is not an +Apache-only distribution choice. Likewise, GPL adapter source from this project +cannot simply be copied into Apache-only ModSecurity; the target adapter's +copyright/license terms must be settled independently. diff --git a/docs/jsoncons-sax.md b/docs/jsoncons-sax.md index c6545cb..27f5d89 100644 --- a/docs/jsoncons-sax.md +++ b/docs/jsoncons-sax.md @@ -1,5 +1,8 @@ # jsoncons SAX evaluation +This document records the original jsoncons/test PR snapshot. Subsequent parser +changes and current verification are documented in [the follow-up stack](adapter-stack.md). + This change implements the parser evaluation requested in [ModSecurity issue #3308](https://github.com/owasp-modsecurity/ModSecurity/issues/3308#issuecomment-6142615388). It does not change ModSecurity or claim integration compatibility with its complete request-body processor. diff --git a/docs/validation-results.md b/docs/validation-results.md index e3faf17..925136f 100644 --- a/docs/validation-results.md +++ b/docs/validation-results.md @@ -1,5 +1,8 @@ # Validation results +This document records the original jsoncons/test PR snapshot. Subsequent parser +changes and current verification are documented in [the follow-up stack](adapter-stack.md). + Validated in `/workspace/digitalwave-jsonbench` based on upstream commit `a89f2e93fb7f1415095d6fb6c1bb5fab6bdc8e6d`. The previous onboarding fork in `/workspace/jsonbench` was left unchanged. Platform: Debian 13 amd64, GCC 14.2, diff --git a/src/jsonbench.c b/src/jsonbench.c index 12fdbdf..7d70a2f 100644 --- a/src/jsonbench.c +++ b/src/jsonbench.c @@ -277,14 +277,21 @@ int main(int argc, char ** argv) { if (n > 1 && !silence) { yajl_json_data *json = NULL; + rc = 1; yajl_json_init(&json, &error_msg); if (json != NULL) { json->depth_limit = depth_limit; json->arg_num_limit = arg_limit; json->silence = 0; - yajl_parse_file(json, jsonfile, &error_msg); + rc = yajl_parse_file(json, jsonfile, &error_msg); yajl_json_cleanup(json); } + if (rc != 0) { + if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } + free(times); + free(jsonengine); + return EXIT_FAILURE; + } } } #endif @@ -331,14 +338,21 @@ int main(int argc, char ** argv) { if (n > 1 && !silence) { rj_parser *json = NULL; + rc = 1; rj_json_init(&json, &error_msg); if (json != NULL) { rj_set_max_depth(json, depth_limit); rj_set_max_arg_num(json, arg_limit); rj_set_silence(json, 0); - rj_parse_file(json, jsonfile, &error_msg); + rc = rj_parse_file(json, jsonfile, &error_msg); rj_json_cleanup(json); } + if (rc != 0) { + if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } + free(times); + free(jsonengine); + return EXIT_FAILURE; + } } } #endif @@ -385,14 +399,21 @@ int main(int argc, char ** argv) { if (n > 1 && !silence) { nl_parser *json = NULL; + rc = 1; nl_json_init(&json, &error_msg); if (json != NULL) { nl_set_max_depth(json, depth_limit); nl_set_max_arg_num(json, arg_limit); nl_set_silence(json, 0); - nl_parse_file(json, jsonfile, &error_msg); + rc = nl_parse_file(json, jsonfile, &error_msg); nl_json_cleanup(json); } + if (rc != 0) { + if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } + free(times); + free(jsonengine); + return EXIT_FAILURE; + } } } #endif @@ -463,14 +484,21 @@ int main(int argc, char ** argv) { if (n > 1 && !silence) { jsonc_parser *json = NULL; + rc = 1; jsonc_json_init(&json, &error_msg); if (json != NULL) { jsonc_set_max_depth(json, depth_limit); jsonc_set_max_arg_num(json, arg_limit); jsonc_set_silence(json, 0); - jsonc_parse_file(json, jsonfile, &error_msg); + rc = jsonc_parse_file(json, jsonfile, &error_msg); jsonc_json_cleanup(json); } + if (rc != 0) { + if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } + free(times); + free(jsonengine); + return EXIT_FAILURE; + } } } #endif @@ -541,14 +569,21 @@ int main(int argc, char ** argv) { if (n > 1 && !silence) { jansson_parser *json = NULL; + rc = 1; jansson_json_init(&json, &error_msg); if (json != NULL) { jansson_set_max_depth(json, depth_limit); jansson_set_max_arg_num(json, arg_limit); jansson_set_silence(json, 0); - jansson_parse_file(json, jsonfile, &error_msg); + rc = jansson_parse_file(json, jsonfile, &error_msg); jansson_json_cleanup(json); } + if (rc != 0) { + if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } + free(times); + free(jsonengine); + return EXIT_FAILURE; + } } } #endif @@ -619,14 +654,21 @@ int main(int argc, char ** argv) { if (n > 1 && !silence) { cjson_parser *json = NULL; + rc = 1; cjson_json_init(&json, &error_msg); if (json != NULL) { cjson_set_max_depth(json, depth_limit); cjson_set_max_arg_num(json, arg_limit); cjson_set_silence(json, 0); - cjson_parse_file(json, jsonfile, &error_msg); + rc = cjson_parse_file(json, jsonfile, &error_msg); cjson_json_cleanup(json); } + if (rc != 0) { + if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } + free(times); + free(jsonengine); + return EXIT_FAILURE; + } } } #endif @@ -697,14 +739,21 @@ int main(int argc, char ** argv) { if (n > 1 && !silence) { jsoncpp_parser *json = NULL; + rc = 1; jsoncpp_json_init(&json, &error_msg); if (json != NULL) { jsoncpp_set_max_depth(json, depth_limit); jsoncpp_set_max_arg_num(json, arg_limit); jsoncpp_set_silence(json, 0); - jsoncpp_parse_file(json, jsonfile, &error_msg); + rc = jsoncpp_parse_file(json, jsonfile, &error_msg); jsoncpp_json_cleanup(json); } + if (rc != 0) { + if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } + free(times); + free(jsonengine); + return EXIT_FAILURE; + } } } #endif @@ -861,14 +910,21 @@ int main(int argc, char ** argv) { if (n > 1 && !silence) { simdjson_parser *json = NULL; + rc = 1; simdjson_json_init(&json, &error_msg); if (json != NULL) { simdjson_set_max_depth(json, depth_limit); simdjson_set_max_arg_num(json, arg_limit); simdjson_set_silence(json, 0); - simdjson_parse_file(json, jsonfile, &error_msg); + rc = simdjson_parse_file(json, jsonfile, &error_msg); simdjson_json_cleanup(json); } + if (rc != 0) { + if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } + free(times); + free(jsonengine); + return EXIT_FAILURE; + } } } #endif @@ -939,14 +995,21 @@ int main(int argc, char ** argv) { if (n > 1 && !silence) { yyjson_parser *json = NULL; + rc = 1; yyjson_json_init(&json, &error_msg); if (json != NULL) { yyjson_set_max_depth(json, depth_limit); yyjson_set_max_arg_num(json, arg_limit); yyjson_set_silence(json, 0); - yyjson_parse_file(json, jsonfile, &error_msg); + rc = yyjson_parse_file(json, jsonfile, &error_msg); yyjson_json_cleanup(json); } + if (rc != 0) { + if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } + free(times); + free(jsonengine); + return EXIT_FAILURE; + } } } #endif @@ -1017,19 +1080,30 @@ int main(int argc, char ** argv) { if (n > 1 && !silence) { glaze_parser *json = NULL; + rc = 1; glaze_json_init(&json, &error_msg); if (json != NULL) { glaze_set_max_depth(json, depth_limit); glaze_set_max_arg_num(json, arg_limit); glaze_set_silence(json, 0); - glaze_parse_file(json, jsonfile, &error_msg); + rc = glaze_parse_file(json, jsonfile, &error_msg); glaze_json_cleanup(json); } + if (rc != 0) { + if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } + free(times); + free(jsonengine); + return EXIT_FAILURE; + } } } #endif free(times); free(jsonengine); + if (fflush(stdout) != 0) { + fprintf(stderr, "Error: Unable to write benchmark output\n"); + return EXIT_FAILURE; + } return 0; } diff --git a/src/pathhandler.h b/src/pathhandler.h index ba2ea70..55c7e4b 100644 --- a/src/pathhandler.h +++ b/src/pathhandler.h @@ -1,120 +1,94 @@ +#ifndef JSONBENCH_PATHHANDLER_H +#define JSONBENCH_PATHHANDLER_H + #include #include +#include +#include +#include +#include -#include "config.h" // needs for JSON_STRING_SIZE (default 8192) +#include "../config.h" +// Preserve structural boundaries: dots and NULs in keys are data, not delimiters +// to search when restoring a parent. The display convention remains historical +// dotted paths with repeated array keys and no numeric array indexes. class PathHandler { - public: - bool m_error; - PathHandler() { - m_prefix[0] = '\0'; - m_current_key[0] = '\0'; - m_prefix_len = 0; - m_current_key_len = 0; - m_error = false; - } + struct Frame { + std::size_t prefix_length; + std::string key; + }; + std::string prefix_; + std::string key_; + std::vector frames_; - // set object key - void setKey(const char* key, size_t len) { - if (m_current_key_len + len >= JSON_STRING_SIZE) { + void start(bool array) { + if (m_error) return; + const bool root_array = array && frames_.empty(); + const std::size_t length = root_array ? 5 : + prefix_.size() + (prefix_.empty() ? 0 : 1) + key_.size(); + if (length >= JSON_STRING_SIZE) { m_error = true; return; } - std::memcpy(m_current_key, key, len); - m_current_key[len] = '\0'; - m_current_key_len = len; - } - - // start_object '{' / start_array '[' - void push() { - if (m_prefix_len == 0) { - std::memcpy(m_prefix, m_current_key, m_current_key_len); - m_prefix_len = m_current_key_len; + frames_.push_back({prefix_.size(), key_}); + if (root_array) { + prefix_ = "array"; + key_ = "array"; } else { - if (m_prefix_len + 1 + m_current_key_len >= JSON_STRING_SIZE) { - // new key would be too long - m_error = true; - return; - } - m_prefix[m_prefix_len++] = '.'; - std::memcpy(m_prefix + m_prefix_len, m_current_key, m_current_key_len); - m_prefix_len += m_current_key_len; + if (!prefix_.empty()) prefix_ += '.'; + prefix_ += key_; } - m_prefix[m_prefix_len] = '\0'; } - // end_object '}' / end_array ']' - void pop() { - if (m_prefix_len == 0) return; - - size_t separator = m_prefix_len; - for (int i = static_cast(m_prefix_len) - 1; i >= 0; --i) { - if (m_prefix[i] == '.') { - separator = static_cast(i); - break; - } - } - - if (separator < m_prefix_len) { - size_t new_len = m_prefix_len - separator - 1; - std::memcpy(m_current_key, m_prefix + separator + 1, new_len); - m_current_key[new_len] = '\0'; - m_current_key_len = new_len; - - m_prefix[separator] = '\0'; - m_prefix_len = separator; - } else { - std::memcpy(m_current_key, m_prefix, m_prefix_len); - m_current_key[m_prefix_len] = '\0'; - m_current_key_len = m_prefix_len; + public: + bool m_error = false; - m_prefix[0] = '\0'; - m_prefix_len = 0; + void setKey(const char* key, std::size_t length) noexcept { + if (m_error) return; + if (length >= JSON_STRING_SIZE || (key == nullptr && length != 0)) { + m_error = true; + return; } + try { key_.assign(key ? key : "", length); } + catch (const std::exception&) { m_error = true; } } - - // array - special case because we use 'array' prefix in case of v2 - void startArraySpecial() { - if (m_prefix_len == 0 && m_current_key_len == 0) { - std::memcpy(m_prefix, "array", 5); - m_prefix_len = 5; - std::memcpy(m_current_key, "array", 5); - m_current_key_len = 5; - m_prefix[5] = '\0'; - m_current_key[5] = '\0'; - } else { - push(); // normal push + void push() noexcept { + try { start(false); } + catch (const std::exception&) { m_error = true; } + } + void startArraySpecial() noexcept { + try { start(true); } + catch (const std::exception&) { m_error = true; } + } + void pop() noexcept { + if (m_error) return; + if (frames_.empty()) { + m_error = true; + return; } + prefix_.resize(frames_.back().prefix_length); + key_ = std::move(frames_.back().key); + frames_.pop_back(); } - - // build argument name from m_prefix - void buildArgName(char* out) { - if (m_prefix_len > 0) { - if (m_prefix_len + m_current_key_len + 1 > JSON_STRING_SIZE) { - m_error = true; - return; - } - std::memcpy(out, m_prefix, m_prefix_len); - std::memcpy(out + m_prefix_len, ".", 1); - std::memcpy(out + m_prefix_len + 1, m_current_key, m_current_key_len); - out[m_prefix_len + 1 + m_current_key_len] = '\0'; - } else { - if (m_current_key_len > JSON_STRING_SIZE) { - m_error = true; - return; - } - std::memcpy(out, m_current_key, m_current_key_len); - out[m_current_key_len] = '\0'; + std::string name() { + const std::size_t length = prefix_.size() + + (prefix_.empty() ? 0 : 1) + key_.size(); + if (m_error || length >= JSON_STRING_SIZE) { + m_error = true; + return {}; } + return prefix_.empty() ? key_ : prefix_ + '.' + key_; + } + // Compatibility for older callers; out must have JSON_STRING_SIZE bytes. + void buildArgName(char* out) noexcept { + try { + const std::string value = name(); + if (m_error || out == nullptr) { m_error = true; return; } + std::memcpy(out, value.data(), value.size()); + out[value.size()] = '\0'; + } catch (const std::exception&) { m_error = true; } } - - // may be these will be needed later... - // const char* getPrefix() const { return m_prefix; } - // size_t getPrefixLen() const { return m_prefix_len; } - - private: - char m_prefix[JSON_STRING_SIZE]; - size_t m_prefix_len; - char m_current_key[JSON_STRING_SIZE]; - size_t m_current_key_len; }; + +#endif diff --git a/src/yajlparser.c b/src/yajlparser.c index 0ecb959..c2a1dfd 100644 --- a/src/yajlparser.c +++ b/src/yajlparser.c @@ -2,426 +2,330 @@ #if HAVE_YAJL -#include +#include #include #include #include -const char *base_offset=NULL; +struct yajl_path_frame { + size_t prefix_length; + size_t key_length; + unsigned char *key; +}; -static int json_add_argument(yajl_json_data *p, const unsigned char *value, unsigned length) -{ - assert(p != NULL); +static int reject(yajl_json_data *p, const char *message) { + p->callback_error = message; + return 0; +} - p->current_arg_num++; - if (p->current_arg_num > p->arg_num_limit) { +static int json_add_argument(yajl_json_data *p, const unsigned char *value, size_t length) { + if (p->current_arg_num >= p->arg_num_limit) { p->arg_num_limit_exceeded = 1; - return 0; - } - - if (p->silence) { - return 1; + return reject(p, "Argument number limit exceeded"); } - - char argname[JSON_STRING_SIZE]; - char argval[JSON_STRING_SIZE]; - - /** - * Argument name is 'prefix + current_key' - */ - if (p->prefix_len > 0) { - if (p->prefix_len + 1 + p->current_key_len >= JSON_STRING_SIZE) { - printf("Argument name too long\n"); - return 0; - } - memcpy(argname, p->prefix, p->prefix_len); - memcpy(argname + p->prefix_len, ".", 1); - memcpy(argname + p->prefix_len + 1, p->current_key, p->current_key_len); - argname[p->prefix_len + 1 + p->current_key_len] = '\0'; + size_t name_length = p->prefix_len + (p->prefix_len ? 1 : 0) + p->current_key_len; + if (name_length >= JSON_STRING_SIZE) return reject(p, "Argument name too long"); + if (length >= JSON_STRING_SIZE) return reject(p, "Argument value too long"); + ++p->current_arg_num; + if (p->silence) return 1; + int ok = 1; + if (p->prefix_len) { + ok = fwrite(p->prefix, 1, p->prefix_len, stdout) == p->prefix_len; + ok = (fputc('.', stdout) != EOF) && ok; } - else { - if (p->current_key_len >= JSON_STRING_SIZE) { - printf("Argument name too long\n"); - return 0; - } - memcpy(argname, p->current_key, p->current_key_len); - argname[p->current_key_len] = '\0'; - } - if (length >= JSON_STRING_SIZE) { - printf("Argument value too long\n"); - return 0; - } - memcpy(argval, value, length); - argval[length] = '\0'; - printf("%s: %s\n", argname, argval); - - return 1; + ok = (fwrite(p->current_key, 1, p->current_key_len, stdout) == p->current_key_len) && ok; + ok = (fwrite(": ", 1, 2, stdout) == 2) && ok; + ok = (fwrite(value, 1, length, stdout) == length) && ok; + ok = (fputc('\n', stdout) != EOF) && ok; + return ok ? 1 : reject(p, "Unable to write JSON argument output"); } -/** - * yajl callback functions - * For more information on the function signatures and order, check - * http://lloyd.github.com/yajl/yajl-1.0.12/structyajl__callbacks.html - */ - -/** - * Callback for hash key values; we use those to define the variable names - * under ARGS. Whenever we reach a new key, we update the current key value. - */ -static int yajl_map_key(void *ctx, const unsigned char *key, size_t length) -{ - yajl_json_data *p = (yajl_json_data *) ctx; - assert(p != NULL); - unsigned char *safe_key = (unsigned char *) NULL; - - /** - * yajl does not provide us with null-terminated strings, but - * rather expects us to copy the data from the key up to the - * length informed; we create a standalone null-termined copy - * in safe_key - */ - safe_key = (unsigned char *)strndup((char *)key, length); - - (unsigned char*)strcpy((char *)p->current_key, (char *)safe_key); +static int map_key(void *ctx, const unsigned char *key, size_t length) { + yajl_json_data *p = ctx; + if (length >= JSON_STRING_SIZE) return reject(p, "Argument name too long"); + memcpy(p->current_key, key, length); p->current_key[length] = '\0'; p->current_key_len = length; - free(safe_key); return 1; } - -static int yajl_null(void *ctx) -{ - yajl_json_data *p = (yajl_json_data *) ctx; - assert(p != NULL); - - return json_add_argument(p, (unsigned char*)"", 0); +static int null_value(void *ctx) { return json_add_argument(ctx, (const unsigned char *)"", 0); } +static int boolean_value(void *ctx, int value) { + return json_add_argument(ctx, (const unsigned char *)(value ? "true" : "false"), value ? 4 : 5); } - -static int yajl_boolean(void *ctx, int value) -{ - yajl_json_data *p = (yajl_json_data *) ctx; - assert(p != NULL); - - if (value) { - return json_add_argument(p, (unsigned char*)"true", strlen("true")); - } - else { - return json_add_argument(p, (unsigned char*)"false", strlen("false")); - } +static int string_value(void *ctx, const unsigned char *value, size_t length) { + return json_add_argument(ctx, value, length); } - -static int yajl_string(void *ctx, const unsigned char *value, size_t length) -{ - yajl_json_data *p = (yajl_json_data *) ctx; - assert(p != NULL); - - return json_add_argument(p, value, length); +static int number_value(void *ctx, const char *value, size_t length) { + return json_add_argument(ctx, (const unsigned char *)value, length); } -static int yajl_number(void *ctx, const char *value, size_t length) -{ - yajl_json_data *p = (yajl_json_data *) ctx; - assert(p != NULL); - - return json_add_argument(p, (unsigned char*)value, length); -} - -static int yajl_start_array(void *ctx) { - yajl_json_data *p = (yajl_json_data *) ctx; - assert(p != NULL); - - if (p->prefix_len == 0 && p->current_key_len == 0) { +static int start_container(yajl_json_data *p, int array) { + if (p->current_depth >= p->depth_limit) { + p->depth_limit_exceeded = 1; + return reject(p, "JSON depth limit exceeded"); + } + int root_array = array && p->frame_count == 0; + size_t length = root_array ? 5 : + p->prefix_len + (p->prefix_len ? 1 : 0) + p->current_key_len; + if (length >= JSON_STRING_SIZE) return reject(p, "Argument name too long"); + if (p->frame_count == p->frame_capacity) { + if (p->frame_capacity > SIZE_MAX / (2 * sizeof(yajl_path_frame))) + return reject(p, "JSON path allocation limit exceeded"); + size_t capacity = p->frame_capacity ? p->frame_capacity * 2 : 16; + yajl_path_frame *frames = realloc(p->frames, capacity * sizeof(*frames)); + if (!frames) return reject(p, "Unable to allocate JSON path stack"); + p->frames = frames; + p->frame_capacity = capacity; + } + unsigned char *saved = malloc(p->current_key_len + 1); + if (!saved) return reject(p, "Unable to allocate JSON path key"); + memcpy(saved, p->current_key, p->current_key_len); + saved[p->current_key_len] = '\0'; + p->frames[p->frame_count++] = (yajl_path_frame){p->prefix_len, p->current_key_len, saved}; + if (root_array) { memcpy(p->prefix, "array", 5); - p->prefix_len = 5; - p->prefix[p->prefix_len] = '\0'; memcpy(p->current_key, "array", 5); - p->current_key_len = 5; - p->current_key[p->current_key_len] = '\0'; - } - else if (p->prefix_len > 0) { - memcpy(p->prefix + p->prefix_len, ".", 1); - p->prefix_len++; + p->prefix_len = p->current_key_len = 5; + } else { + if (p->prefix_len) p->prefix[p->prefix_len++] = '.'; memcpy(p->prefix + p->prefix_len, p->current_key, p->current_key_len); p->prefix_len += p->current_key_len; - p->prefix[p->prefix_len] = '\0'; - } - else { - memcpy(p->prefix, p->current_key, p->current_key_len); - p->prefix_len += p->current_key_len; - p->prefix[p->prefix_len] = '\0'; - } - - p->current_depth++; - if (p->current_depth > p->depth_limit) { - p->depth_limit_exceeded = 1; - return 0; } - + p->prefix[p->prefix_len] = '\0'; + p->current_key[p->current_key_len] = '\0'; + ++p->current_depth; return 1; } - - -static int yajl_end_array(void *ctx) { - yajl_json_data *p = (yajl_json_data *) ctx; - assert(p != NULL); - - size_t separator = p->prefix_len; - - for(int i = p->prefix_len - 1; i >= 0; i--) { - if (p->prefix[i] == '.') { - separator = i; - break; - } - } - - if (separator < p->prefix_len) { - p->prefix[separator] = '\0'; - p->prefix_len = separator; - } - else { - p->prefix[0] = '\0'; - p->prefix_len = 0; - } - p->current_depth--; - +static int start_object(void *ctx) { return start_container(ctx, 0); } +static int start_array(void *ctx) { return start_container(ctx, 1); } +static int end_container(void *ctx) { + yajl_json_data *p = ctx; + if (!p->frame_count) return reject(p, "Unpaired JSON container event"); + yajl_path_frame *frame = &p->frames[--p->frame_count]; + p->prefix_len = frame->prefix_length; + p->prefix[p->prefix_len] = '\0'; + p->current_key_len = frame->key_length; + memcpy(p->current_key, frame->key, frame->key_length + 1); + free(frame->key); + --p->current_depth; return 1; } +static const yajl_callbacks callbacks = { + null_value, boolean_value, NULL, NULL, number_value, string_value, + start_object, map_key, end_container, start_array, end_container +}; -/** - * Callback for a new hash, which indicates a new subtree, labeled as the current - * argument name, is being created - */ -static int yajl_start_map(void *ctx) -{ - yajl_json_data *p = (yajl_json_data *) ctx; - assert(p != NULL); - - if (p->prefix_len == 0) { - memcpy(p->prefix, p->current_key, p->current_key_len); - p->prefix_len = p->current_key_len; - p->prefix[p->prefix_len] = '\0'; - } - else { - memcpy(p->prefix + p->prefix_len, ".", 1); - p->prefix_len++; - memcpy(p->prefix + p->prefix_len, p->current_key, p->current_key_len); - p->prefix_len += p->current_key_len; - p->prefix[p->prefix_len] = '\0'; - } - p->current_depth++; - if (p->current_depth > p->depth_limit) { - p->depth_limit_exceeded = 1; - return 0; - } - - return 1; +static void clear_state(yajl_json_data *p) { + for (size_t i = 0; i < p->frame_count; ++i) free(p->frames[i].key); + p->frame_count = 0; + p->prefix_len = p->current_key_len = 0; + p->prefix[0] = p->current_key[0] = '\0'; + p->current_depth = p->current_arg_num = 0; + p->depth_limit_exceeded = p->arg_num_limit_exceeded = 0; + p->callback_error = NULL; + p->escape_state = p->escape_digits = p->escape_codepoint = 0; + p->low_surrogate_required = p->number_token = p->completed = 0; + p->token_length = 0; + p->utf8_remaining = 0; + p->utf8_min = 0x80; + p->utf8_max = 0xbf; + p->status = yajl_status_ok; } - -/** - * Callback for end hash, meaning the current subtree is being closed, and that - * we should go back to the parent variable label - */ -static int yajl_end_map(void *ctx) -{ - yajl_json_data *p = (yajl_json_data *) ctx; - assert(p != NULL); - size_t separator = p->prefix_len; - - for(int i = p->prefix_len - 1; i >= 0; i--) { - if (p->prefix[i] == '.') { - separator = i; - break; - } - } - - if (separator < p->prefix_len) { - memcpy(p->current_key, p->prefix + separator + 1, p->prefix_len - separator - 1); - p->current_key_len = p->prefix_len - separator - 1; - p->prefix[separator] = '\0'; - p->prefix_len = separator; - } - else { - memcpy(p->current_key, p->prefix, p->prefix_len); - p->current_key_len = p->prefix_len; - p->prefix[0] = '\0'; - p->prefix_len = 0; +static int reset(yajl_json_data *p, char **error_msg) { + if (p->handle) yajl_free(p->handle); + clear_state(p); + p->handle = yajl_alloc(&callbacks, NULL, p); + if (!p->handle) { + *error_msg = strdup("Unable to allocate YAJL parser"); + return -1; } - p->current_depth--; - + yajl_config(p->handle, yajl_allow_partial_values, 0); + yajl_config(p->handle, yajl_dont_validate_strings, 0); return 1; } - -static yajl_callbacks callbacks = { - yajl_null, - yajl_boolean, - NULL /* yajl_integer */, - NULL /* yajl_double */, - yajl_number, - yajl_string, - yajl_start_map, - yajl_map_key, - yajl_end_map, - yajl_start_array, - yajl_end_array -}; - -/** - * Initialise JSON parser. - */ int yajl_json_init(yajl_json_data **json, char **error_msg) { - assert(error_msg != NULL); - + if (!error_msg) return -1; *error_msg = NULL; - - (*json) = malloc(sizeof(yajl_json_data)); - if (*json == NULL) return -1; - - /** - * Prefix and current key are initially empty - */ - (*json)->prefix = (unsigned char *) malloc(JSON_STRING_SIZE * sizeof(unsigned char)); - if ((*json)->prefix == NULL) return -1; - memset((*json)->prefix, 0, JSON_STRING_SIZE); - (*json)->prefix_len = 0; - (*json)->current_key = (unsigned char *) malloc(JSON_STRING_SIZE * sizeof(unsigned char)); - if ((*json)->current_key == NULL) return -1; - memset((*json)->current_key, 0, JSON_STRING_SIZE); - (*json)->current_key_len = 0; - - (*json)->current_depth = 0; - (*json)->depth_limit = 0; - (*json)->depth_limit_exceeded = 0; - (*json)->current_arg_num = 0; - (*json)->arg_num_limit = 0; - (*json)->arg_num_limit_exceeded = 0; - (*json)->silence = 0; - - /** - * yajl initialization - * - * yajl_parser_config definition: - * http://lloyd.github.io/yajl/yajl-2.0.1/yajl__parse_8h.html#aec816c5518264d2ac41c05469a0f986c - * - * TODO: make UTF8 validation optional, as it depends on Content-Encoding - */ - - (*json)->handle = yajl_alloc(&callbacks, NULL, (*json)); - yajl_config((*json)->handle, yajl_allow_partial_values, 0); - + if (!json) return -1; + *json = NULL; + yajl_json_data *p = calloc(1, sizeof(*p)); + if (!p) { *error_msg = strdup("Unable to allocate YAJL parser"); return -1; } + p->prefix = malloc(JSON_STRING_SIZE); + p->current_key = malloc(JSON_STRING_SIZE); + p->depth_limit = p->arg_num_limit = 500; + if (!p->prefix || !p->current_key || reset(p, error_msg) < 0) { + if (!*error_msg) *error_msg = strdup("Unable to allocate YAJL path buffers"); + yajl_json_cleanup(p); + return -1; + } + *json = p; return 1; } -/** - * Feed one chunk of data to the JSON parser. - */ -int yajl_json_process_chunk(yajl_json_data *json, const char *buf, unsigned int size, char **error_msg) { - assert(json != NULL); - assert(error_msg != NULL); - *error_msg = NULL; - base_offset=buf; - - /* Feed our parser and catch any errors */ - json->status = yajl_parse(json->handle, (unsigned char *)buf, size); - - if (json->status != yajl_status_ok) { - if (json->depth_limit_exceeded != 0 || json->arg_num_limit_exceeded != 0) { - if (json->depth_limit_exceeded != 0) { - const char * err = "JSON depth limit exceeded"; - *error_msg = strdup(err); - } - if (json->arg_num_limit_exceeded != 0) { - const char * err = "ARGUMENT number limit exceeded"; - if (*error_msg == NULL) { - *error_msg = strdup(err); - } - else { - *error_msg = realloc(*error_msg, strlen(*error_msg) + strlen(err) + 1); - strcat(*error_msg, err); +// Narrow incremental token/Unicode guard. YAJL 2.1 does not validate that an +// escaped surrogate pair's second unit is a low surrogate. It also grows lexer +// buffers before callbacks can reject oversized values. JSON grammar stays YAJL's. +static int check_input(yajl_json_data *p, const char *buf, size_t length) { + const size_t budget = (size_t)JSON_STRING_SIZE * 8 + 16; + for (size_t i = 0; i < length; ++i) { + unsigned char c = (unsigned char)buf[i]; + if (!c) return reject(p, "Invalid raw NUL byte in JSON input"); + // YAJL 2.1's native check accepts overlong encodings and surrogate + // codepoints; enforce Unicode scalar UTF-8 independently across chunks. + if (p->utf8_remaining) { + if (c < p->utf8_min || c > p->utf8_max) + return reject(p, "Invalid UTF-8 sequence"); + --p->utf8_remaining; + p->utf8_min = 0x80; + p->utf8_max = 0xbf; + } else if (c >= 0x80) { + p->utf8_min = 0x80; + p->utf8_max = 0xbf; + if (c >= 0xc2 && c <= 0xdf) p->utf8_remaining = 1; + else if (c >= 0xe0 && c <= 0xef) { + p->utf8_remaining = 2; + if (c == 0xe0) p->utf8_min = 0xa0; + if (c == 0xed) p->utf8_max = 0x9f; + } else if (c >= 0xf0 && c <= 0xf4) { + p->utf8_remaining = 3; + if (c == 0xf0) p->utf8_min = 0x90; + if (c == 0xf4) p->utf8_max = 0x8f; + } else return reject(p, "Invalid UTF-8 sequence"); + } + if (p->escape_state) { + if (++p->token_length > budget) return reject(p, "JSON token allocation limit exceeded"); + switch (p->escape_state) { + case 1: + if (c == '"') { p->escape_state = 0; p->token_length = 0; } + else if (c == '\\') p->escape_state = 2; + break; + case 2: + if (c == 'u') { + p->escape_state = 3; + p->escape_digits = p->escape_codepoint = 0; + } else p->escape_state = 1; + break; + case 3: { + unsigned int digit; + if (c >= '0' && c <= '9') digit = c - '0'; + else if (c >= 'a' && c <= 'f') digit = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') digit = c - 'A' + 10; + else return reject(p, "Invalid Unicode escape"); + p->escape_codepoint = p->escape_codepoint * 16 + digit; + if (++p->escape_digits < 4) break; + if (p->low_surrogate_required) { + if (p->escape_codepoint < 0xdc00 || p->escape_codepoint > 0xdfff) + return reject(p, "Invalid Unicode surrogate escape"); + p->low_surrogate_required = 0; + p->escape_state = 1; + } else if (p->escape_codepoint >= 0xd800 && p->escape_codepoint <= 0xdbff) { + p->low_surrogate_required = 1; + p->escape_state = 4; + } else { + if (p->escape_codepoint >= 0xdc00 && p->escape_codepoint <= 0xdfff) + return reject(p, "Invalid Unicode surrogate escape"); + p->escape_state = 1; } + break; + } + case 4: + if (c != '\\') return reject(p, "Invalid Unicode surrogate escape"); + p->escape_state = 5; + break; + case 5: + if (c != 'u') return reject(p, "Invalid Unicode surrogate escape"); + p->escape_state = 3; + p->escape_digits = p->escape_codepoint = 0; + break; + } + } else if (c == '"') { + p->number_token = 0; + p->escape_state = 1; + p->token_length = 1; + } else { + int number_char = (c >= '0' && c <= '9') || c == '-' || c == '+' || c == '.' || c == 'e' || c == 'E'; + if (p->number_token && number_char) { + if (++p->token_length > budget) return reject(p, "JSON token allocation limit exceeded"); + } else { + p->number_token = (c >= '0' && c <= '9') || c == '-'; + p->token_length = p->number_token ? 1 : 0; } } - else { - unsigned char* yajl_err = yajl_get_error(json->handle, 0, (unsigned char *)buf, size); - *error_msg = strdup((char *)yajl_err); - yajl_free_error(json->handle, yajl_err); - } - return -1; } - return 1; } - -/** - * Frees the resources used for JSON parsing. - */ -int yajl_json_cleanup(yajl_json_data *json) { - assert(json != NULL); - if (json->handle != NULL) { - yajl_free(json->handle); - json->handle = NULL; - } - free(json->prefix); - free(json->current_key); - free(json); - - return 1; +static int status_result(yajl_json_data *p, const char *buf, size_t length, char **error_msg) { + if (p->callback_error) *error_msg = strdup(p->callback_error); + else if (p->status != yajl_status_ok) { + unsigned char *message = yajl_get_error(p->handle, 0, (const unsigned char *)buf, length); + *error_msg = strdup(message ? (const char *)message : "YAJL parsing failed"); + if (message) yajl_free_error(p->handle, message); + } else return 1; + return -1; } - -#ifndef FILE_BUFFER_SIZE -#define FILE_BUFFER_SIZE 10485760 -#endif - -int yajl_parse_file(yajl_json_data *json, const char *filename, char **error_msg) { - assert(json != NULL); - assert(filename != NULL); - assert(error_msg != NULL); - - FILE *fp = fopen(filename, "r"); - if (fp == NULL) { - *error_msg = strdup("Unable to open file"); - return -1; - } - - char *buf = (char *)malloc(FILE_BUFFER_SIZE); - if (buf == NULL) { - fclose(fp); - *error_msg = strdup("Memory allocation error"); - return -1; - } - - size_t len = fread(buf, 1, FILE_BUFFER_SIZE - 1, fp); - int read_error = ferror(fp); - int too_long = 0; - if (!read_error && len == FILE_BUFFER_SIZE - 1) { - int extra = fgetc(fp); - if (extra != EOF) { - too_long = 1; - } else { - read_error = ferror(fp); - } +int yajl_json_process_chunk(yajl_json_data *p, const char *buf, unsigned int length, char **error_msg) { + if (!error_msg) return -1; + *error_msg = NULL; + if (!p || !p->handle || (!buf && length)) return -1; + if (p->completed) { *error_msg = strdup("YAJL document is already finalized"); return -1; } + if (p->callback_error || !check_input(p, buf, length)) return status_result(p, buf, length, error_msg); + p->status = yajl_parse(p->handle, (const unsigned char *)(buf ? buf : ""), length); + return status_result(p, buf, length, error_msg); +} +int yajl_json_process(yajl_json_data *p, const char *buf, unsigned int length, char **error_msg) { + return yajl_json_process_chunk(p, buf, length, error_msg); +} +int yajl_json_complete(yajl_json_data *p, char **error_msg) { + if (!error_msg) return -1; + *error_msg = NULL; + if (!p || !p->handle) return -1; + if (p->completed) { *error_msg = strdup("YAJL document is already finalized"); return -1; } + if (p->callback_error) return status_result(p, "", 0, error_msg); + if (p->utf8_remaining) { + reject(p, "Truncated UTF-8 sequence"); + return status_result(p, "", 0, error_msg); } - fclose(fp); - - if (read_error) { - free(buf); - *error_msg = strdup("Error reading file"); + p->status = yajl_complete_parse(p->handle); + p->completed = 1; + if (status_result(p, "", 0, error_msg) < 0) return -1; + if (!p->silence && fflush(stdout) != 0) { + *error_msg = strdup("Unable to write JSON argument output"); return -1; } - if (too_long) { - free(buf); - *error_msg = strdup("File too long"); + return 1; +} +int yajl_json_cleanup(yajl_json_data *p) { + if (!p) return 1; + if (p->handle) yajl_free(p->handle); + for (size_t i = 0; i < p->frame_count; ++i) free(p->frames[i].key); + free(p->frames); + free(p->prefix); + free(p->current_key); + free(p); + return 1; +} +int yajl_parse_file(yajl_json_data *p, const char *filename, char **error_msg) { + if (!error_msg) return -1; + *error_msg = NULL; + if (!p || !filename) return -1; + if (p->depth_limit < 0 || p->arg_num_limit < 0) { + *error_msg = strdup("Invalid negative YAJL parser limit"); return -1; } - if (len == 0) { - free(buf); - *error_msg = strdup("Zero bytes read from file"); - return -1; + FILE *input = fopen(filename, "rb"); + if (!input) { *error_msg = strdup("Unable to open JSON file"); return -1; } + if (reset(p, error_msg) < 0) { fclose(input); return -1; } + char buffer[16384]; + size_t length; + int rc = 1; + while ((length = fread(buffer, 1, sizeof(buffer), input)) != 0) { + rc = yajl_json_process_chunk(p, buffer, (unsigned int)length, error_msg); + if (rc < 0) break; } - buf[len] = '\0'; - - int rc = yajl_json_process_chunk(json, buf, (unsigned int)len, error_msg); - free(buf); - return rc == 1 ? 0 : -1; + if (rc > 0 && ferror(input)) { *error_msg = strdup("Error reading JSON file"); rc = -1; } + fclose(input); + if (rc > 0) rc = yajl_json_complete(p, error_msg); + return rc > 0 ? 0 : -1; } #endif diff --git a/src/yajlparser.h b/src/yajlparser.h index 1a2c947..5cb9a75 100644 --- a/src/yajlparser.h +++ b/src/yajlparser.h @@ -6,6 +6,7 @@ #if HAVE_YAJL typedef struct yajl_json_data yajl_json_data; +typedef struct yajl_path_frame yajl_path_frame; #include @@ -27,6 +28,21 @@ struct yajl_json_data { long int arg_num_limit; // cppcheck-suppress unusedStructMember int arg_num_limit_exceeded; // cppcheck-suppress unusedStructMember int silence; // cppcheck-suppress unusedStructMember + /* Structural path boundaries and incremental lexical guards. */ + yajl_path_frame *frames; + size_t frame_count; + size_t frame_capacity; + const char *callback_error; + unsigned int escape_state; + unsigned int escape_digits; + unsigned int escape_codepoint; + int low_surrogate_required; + size_t token_length; + int number_token; + int completed; + unsigned int utf8_remaining; + unsigned int utf8_min; + unsigned int utf8_max; }; #ifdef __cplusplus diff --git a/tests/pathhandler_tests.cpp b/tests/pathhandler_tests.cpp new file mode 100644 index 0000000..da0d70b --- /dev/null +++ b/tests/pathhandler_tests.cpp @@ -0,0 +1,217 @@ +// Public path-builder contracts shared by SAX adapters. These tests do not +// depend on private stack layout or one particular parser implementation. +#include "../src/pathhandler.h" + +#include +#include +#include +#include +#include + +namespace { +class Checks { + unsigned int total_ = 0; + unsigned int failures_ = 0; + public: + void check(const std::string& name, bool passed) { + ++total_; + if (!passed) { + ++failures_; + std::fprintf(stderr, "FAIL: %s\n", name.c_str()); + } + } + void path(const std::string& label, PathHandler& handler, const std::string& expected) { + check(label + ": byte-exact name", handler.name() == expected); + check(label + ": no path error", !handler.m_error); + // Both NUL terminators and bytes beyond the declared destination size + // matter: oversized writes must not corrupt adjacent memory. + std::vector buffer(JSON_STRING_SIZE + 16, '#'); + handler.buildArgName(buffer.data() + 8); + check(label + ": legacy output bytes", std::memcmp(buffer.data() + 8, + expected.data(), expected.size()) == 0); + check(label + ": legacy output terminator", buffer[8 + expected.size()] == '\0'); + bool canaries = true; + for (unsigned int i = 0; i < 8; ++i) + canaries = canaries && buffer[i] == '#' && buffer[JSON_STRING_SIZE + 8 + i] == '#'; + check(label + ": destination canaries unchanged", canaries); + } + int finish() const { + std::printf("PathHandler: %u checks passed, %u failed\n", total_ - failures_, failures_); + return failures_ ? 1 : 0; + } +}; + +void structural_paths(Checks& checks) { + PathHandler handler; + checks.path("root scalar has empty name", handler, ""); + handler.push(); // Root object. + handler.setKey("a.b", 3); + handler.push(); + handler.setKey("c.d", 3); + checks.path("dots inside both keys", handler, "a.b.c.d"); + handler.push(); + handler.setKey("leaf", 4); + checks.path("three levels", handler, "a.b.c.d.leaf"); + handler.pop(); + checks.path("pop restores full dotted parent key", handler, "a.b.c.d"); + handler.setKey("next", 4); + checks.path("nested sibling after dotted key", handler, "a.b.next"); + handler.pop(); + checks.path("pop restores outer dotted key", handler, "a.b"); + handler.setKey("after", 5); + checks.path("root sibling after nested objects", handler, "after"); + handler.pop(); + checks.path("root pop restores empty state", handler, ""); + + PathHandler arrays; + arrays.startArraySpecial(); + checks.path("root array convention", arrays, "array.array"); + arrays.startArraySpecial(); + checks.path("nested root array repeats key", arrays, "array.array.array"); + arrays.pop(); + checks.path("nested array restores parent", arrays, "array.array"); + arrays.push(); + arrays.setKey("key.with.dot", 12); + checks.path("object inside root array", arrays, "array.array.key.with.dot"); + arrays.pop(); + checks.path("object pop restores array scalar path", arrays, "array.array"); + arrays.pop(); + checks.path("root array pop restores root", arrays, ""); + + PathHandler mixed; + mixed.push(); + mixed.setKey("a.b", 3); + mixed.startArraySpecial(); + checks.path("named array duplicates whole dotted key", mixed, "a.b.a.b"); + mixed.push(); + mixed.setKey("child", 5); + checks.path("object in named array", mixed, "a.b.a.b.child"); + mixed.pop(); + checks.path("object in array restores whole dotted key", mixed, "a.b.a.b"); + mixed.pop(); + mixed.setKey("sibling", 7); + checks.path("named array restores containing object", mixed, "sibling"); + mixed.pop(); + checks.path("mixed structure returns to root", mixed, ""); +} + +void empty_and_binary_keys(Checks& checks) { + PathHandler empty; + empty.push(); + empty.setKey(nullptr, 0); + checks.path("null pointer with zero-length key", empty, ""); + empty.push(); + empty.setKey("", 0); + checks.path("nested empty keys follow display convention", empty, ""); + empty.setKey("x", 1); + checks.path("child under empty parent", empty, "x"); + empty.pop(); + checks.path("empty parent restored", empty, ""); + empty.pop(); + checks.path("empty root restored", empty, ""); + + PathHandler binary; + const char parent[] = {'a', '\0', 'b'}; + const char child[] = {'c', '\0', 'd'}; + const std::string parent_string(parent, sizeof parent); + const std::string child_string(child, sizeof child); + binary.push(); + binary.setKey(parent, sizeof parent); + binary.push(); + binary.setKey(child, sizeof child); + checks.path("embedded NUL keys preserve bytes", binary, parent_string + "." + child_string); + binary.push(); + binary.setKey("leaf", 4); + checks.path("embedded NUL nested prefix", binary, parent_string + "." + child_string + ".leaf"); + binary.pop(); + checks.path("embedded NUL nested parent restored", binary, parent_string + "." + child_string); + binary.pop(); + checks.path("embedded NUL outer parent restored", binary, parent_string); + binary.setKey("after", 5); + checks.path("sibling after embedded NUL container", binary, "after"); + binary.pop(); + checks.path("binary-key structure returns to root", binary, ""); +} + +void boundaries_and_errors(Checks& checks) { + const std::size_t limit = JSON_STRING_SIZE; + const std::string max_key(limit - 1, 'k'); + PathHandler key; + key.setKey(max_key.data(), max_key.size()); + checks.path("key exactly below byte limit", key, max_key); + const std::string oversized_key = max_key + "k"; + key.setKey(oversized_key.data(), oversized_key.size()); + checks.check("key at byte limit rejected", key.m_error); + checks.check("errored key has empty name", key.name().empty()); + key.setKey("safe", 4); + key.push(); + key.pop(); + checks.check("path error stays sticky", key.m_error && key.name().empty()); + + const std::string parent((limit - 2) / 2, 'p'); + const std::string child(limit - 2 - parent.size(), 'c'); + PathHandler exact; + exact.setKey(parent.data(), parent.size()); + exact.push(); + exact.setKey(child.data(), child.size()); + checks.path("full path exactly below byte limit", exact, parent + "." + child); + exact.push(); + // Entering this container is legal; a further nonempty leaf cannot fit. + exact.setKey("x", 1); + checks.check("leaf above full path limit rejected", exact.name().empty() && exact.m_error); + + PathHandler too_long; + too_long.setKey(parent.data(), parent.size()); + too_long.push(); + const std::string longer_child = child + "c"; + too_long.setKey(longer_child.data(), longer_child.size()); + checks.check("full path at byte limit rejected", too_long.name().empty() && too_long.m_error); + std::vector untouched(limit, '!'); + too_long.buildArgName(untouched.data()); + checks.check("errored legacy conversion writes no bytes", + untouched == std::vector(limit, '!')); + + PathHandler push_limit; + push_limit.setKey(parent.data(), parent.size()); + push_limit.push(); + push_limit.setKey(longer_child.data(), longer_child.size()); + push_limit.push(); + checks.check("oversized container prefix rejected on push", push_limit.m_error); + + PathHandler null_key; + null_key.setKey(nullptr, 1); + checks.check("null nonempty key rejected", null_key.m_error); + PathHandler null_output; + null_output.buildArgName(nullptr); + checks.check("null output destination rejected", null_output.m_error); + PathHandler underflow; + underflow.pop(); + checks.check("unpaired pop rejected", underflow.m_error); + + PathHandler many; + many.push(); + const unsigned int depth = 1000; + for (unsigned int i = 0; i < depth; ++i) { + many.setKey("x", 1); + many.push(); + } + checks.check("deep structural stack valid", !many.m_error); + for (unsigned int i = 0; i < depth; ++i) many.pop(); + checks.path("deep stack restores root object key", many, "x"); + many.pop(); + checks.path("deep stack returns to scalar root", many, ""); +} +} // namespace + +int main() { + try { + Checks checks; + structural_paths(checks); + empty_and_binary_keys(checks); + boundaries_and_errors(checks); + return checks.finish(); + } catch (const std::exception& error) { + std::fprintf(stderr, "FAIL: PathHandler test setup: %s\n", error.what()); + return 1; + } +} diff --git a/tests/regression_tests.py b/tests/regression_tests.py index 303a960..8ea5dcd 100755 --- a/tests/regression_tests.py +++ b/tests/regression_tests.py @@ -50,9 +50,9 @@ def utf8(text): b"s: text\nt: true\nf: false\nn: \ni: -42\nd: 1.25\n", 6), Case("root-string", b'"hello"', b": hello\n", 1), Case("root-integer", b"42\n", b": 42\n", 1), - Case("root-integer-eof", b"42", b": 42\n", 1, b""), + Case("root-integer-eof", b"42", b": 42\n", 1), Case("root-decimal", b"1.25\n", b": 1.25\n", 1), - Case("root-decimal-eof", b"1.25", b": 1.25\n", 1, b""), + Case("root-decimal-eof", b"1.25", b": 1.25\n", 1), Case("root-true", b"true", b": true\n", 1), Case("root-false", b"false", b": false\n", 1), Case("root-null", b"null", b": \n", 1), @@ -73,7 +73,7 @@ def utf8(text): b"k: 1\nk: 2\nnest.k: 3\nnest.k: 4\n", 4), Case("empty-keys", b'{"":{"":1},"last":2}', b": 1\nlast: 2\n", 2), Case("dotted-keys", b'{"a.b":{"c":1},"d":2}', - b"a.b.c: 1\nd: 2\n", 2, b"a.b.c: 1\na.d: 2\n"), + b"a.b.c: 1\nd: 2\n", 2), Case("unicode", utf8('{"Grüße":"日本語 😀","escaped":"\\u20ac\\ud83d\\ude00"}'), utf8("Grüße: 日本語 😀\nescaped: €😀\n"), 2), Case("literal-unicode-escape", b'{"literal":"\\\\udc00"}', b"literal: \\udc00\n", 1), @@ -82,7 +82,7 @@ def utf8(text): Case("utf8-across-buffer", b" " * 16381 + utf8('"😀"'), utf8(": 😀\n"), 1), Case("escaped-strings", b'{"s":"quote:\\" slash:\\/ backslash:\\\\ tab:\\t line:\\n end"}', b's: quote:" slash:/ backslash:\\ tab:\t line:\n end\n', 1), - Case("embedded-nul", b'{"a\\u0000b":"x\\u0000y"}', b"a\0b: x\0y\n", 1, b"a: x\n"), + Case("embedded-nul", b'{"a\\u0000b":"x\\u0000y"}', b"a\0b: x\0y\n", 1), Case("timing-like-value", b'{"Time":"0.000000000 sec","blank":"\\n"}', b"Time: 0.000000000 sec\nblank: \n\n", 2), Case("integer-boundaries", diff --git a/tests/yajl_api_tests.cpp b/tests/yajl_api_tests.cpp new file mode 100644 index 0000000..d24805f --- /dev/null +++ b/tests/yajl_api_tests.cpp @@ -0,0 +1,480 @@ +// Public C API regressions, including boundaries unavailable through the CLI. +// Compile the YAJL adapter as C, then link this harness with the enabled YAJL. +#include "../src/yajlparser.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if HAVE_YAJL +namespace { + +class TemporaryInput { + std::string path_; + public: + explicit TemporaryInput(const std::string& text) { + char name[] = "/tmp/jsonbench-yajl-api-XXXXXX"; + const int fd = mkstemp(name); + if (fd < 0) throw std::runtime_error("mkstemp failed"); + path_ = name; + std::size_t offset = 0; + while (offset < text.size()) { + const ssize_t count = write(fd, text.data() + offset, text.size() - offset); + if (count < 0 && errno == EINTR) continue; + if (count <= 0) { + close(fd); + unlink(path_.c_str()); + throw std::runtime_error("temporary input write failed"); + } + offset += static_cast(count); + } + if (close(fd) != 0) { + unlink(path_.c_str()); + throw std::runtime_error("temporary input close failed"); + } + } + ~TemporaryInput() { unlink(path_.c_str()); } + TemporaryInput(const TemporaryInput&) = delete; + TemporaryInput& operator=(const TemporaryInput&) = delete; + const char* path() const { return path_.c_str(); } +}; + +class CapturedOutput { + FILE* file_ = nullptr; + int saved_fd_ = -1; + public: + CapturedOutput() { + if (std::fflush(stdout) != 0) throw std::runtime_error("stdout flush failed"); + file_ = std::tmpfile(); + if (!file_) throw std::runtime_error("stdout temporary file failed"); + saved_fd_ = dup(STDOUT_FILENO); + if (saved_fd_ < 0 || dup2(fileno(file_), STDOUT_FILENO) < 0) { + if (saved_fd_ >= 0) close(saved_fd_); + std::fclose(file_); + throw std::runtime_error("stdout capture failed"); + } + } + ~CapturedOutput() { + std::fflush(stdout); + dup2(saved_fd_, STDOUT_FILENO); + close(saved_fd_); + std::fclose(file_); + } + CapturedOutput(const CapturedOutput&) = delete; + CapturedOutput& operator=(const CapturedOutput&) = delete; + std::string read() { + if (std::fflush(stdout) != 0 || std::fseek(file_, 0, SEEK_SET) != 0) + throw std::runtime_error("captured output rewind failed"); + std::string result; + char buffer[1024]; + std::size_t length; + while ((length = std::fread(buffer, 1, sizeof buffer, file_)) != 0) + result.append(buffer, length); + if (std::ferror(file_)) throw std::runtime_error("captured output read failed"); + return result; + } +}; + +// Callers own both their old diagnostic and each newly returned diagnostic. +// Starting with an allocation also catches APIs incorrectly freeing old errors. +class ErrorSlot { + char* original_; + public: + char* value; + ErrorSlot() : original_(static_cast(std::malloc(1))), value(original_) { + if (!original_) throw std::bad_alloc(); + *original_ = '\0'; + } + ~ErrorSlot() { + if (value != original_) std::free(value); + std::free(original_); + } + ErrorSlot(const ErrorSlot&) = delete; + ErrorSlot& operator=(const ErrorSlot&) = delete; + bool replaced() const { return value != original_; } +}; + +class Parser { + public: + yajl_json_data* value = nullptr; + Parser() { + ErrorSlot error; + if (yajl_json_init(&value, &error.value) != 1 || !value || error.value) { + yajl_json_cleanup(value); + throw std::runtime_error("parser initialization failed"); + } + } + ~Parser() { yajl_json_cleanup(value); } + Parser(const Parser&) = delete; + Parser& operator=(const Parser&) = delete; +}; + +struct Result { + int status; + bool replaced; + bool null_error; + std::string error; + std::string output; +}; + +Result parse_path(yajl_json_data* parser, const char* path) { + ErrorSlot error; + CapturedOutput output; + const int status = yajl_parse_file(parser, path, &error.value); + return {status, error.replaced(), !error.value, + error.value ? error.value : "", output.read()}; +} + +Result parse(yajl_json_data* parser, const std::string& text) { + TemporaryInput input(text); + return parse_path(parser, input.path()); +} + +class Checks { + unsigned int total_ = 0; + unsigned int failures_ = 0; + public: + void check(const std::string& name, bool passed) { + ++total_; + if (!passed) { + ++failures_; + std::fprintf(stderr, "FAIL: %s\n", name.c_str()); + } + } + void success(const std::string& name, const Result& result, + const std::string& output = "") { + check(name + ": succeeds", result.status == 0); + check(name + ": clears old diagnostic", result.replaced && result.null_error); + check(name + ": byte-exact output", result.output == output); + if (result.status != 0) + std::fprintf(stderr, " diagnostic: %s\n", result.error.c_str()); + } + void failure(const std::string& name, const Result& result) { + check(name + ": fails", result.status == -1); + check(name + ": fresh malloc-owned diagnostic", + result.replaced && !result.null_error && !result.error.empty()); + } + int finish() const { + std::printf("YAJL C API: %u checks passed, %u failed\n", total_ - failures_, failures_); + return failures_ ? 1 : 0; + } +}; + +void invalid_arguments(Checks& checks) { + { + ErrorSlot error; + checks.check("init rejects null parser slot", yajl_json_init(nullptr, &error.value) == -1); + checks.check("invalid init clears old diagnostic", error.replaced() && !error.value); + } + yajl_json_data* parser_slot = nullptr; + checks.check("init rejects null diagnostic slot", yajl_json_init(&parser_slot, nullptr) == -1); + checks.check("invalid init leaves parser null", !parser_slot); + checks.check("init rejects both null slots", yajl_json_init(nullptr, nullptr) == -1); + checks.check("cleanup accepts null", yajl_json_cleanup(nullptr) == 1); + Parser parser; + TemporaryInput input("null"); + for (const auto& invalid : std::vector>{ + {"null parser", parse_path(nullptr, input.path())}, + {"null filename", parse_path(parser.value, nullptr)}}) { + checks.check("file parse rejects " + invalid.first, invalid.second.status == -1); + checks.check("file parse clears diagnostic for " + invalid.first, + invalid.second.replaced && invalid.second.null_error); + } + checks.check("file parse rejects null diagnostic slot", + yajl_parse_file(parser.value, input.path(), nullptr) == -1); + checks.failure("missing file", parse_path(parser.value, (std::string(input.path()) + ".missing").c_str())); + { + ErrorSlot error; + checks.check("chunk rejects null parser", yajl_json_process_chunk(nullptr, "{}", 2, &error.value) == -1); + checks.check("null parser chunk clears diagnostic", error.replaced() && !error.value); + } + { + ErrorSlot error; + checks.check("chunk rejects null nonempty buffer", yajl_json_process_chunk(parser.value, nullptr, 1, &error.value) == -1); + checks.check("null buffer chunk clears diagnostic", error.replaced() && !error.value); + } + checks.check("chunk rejects null diagnostic slot", yajl_json_process_chunk(parser.value, "{}", 2, nullptr) == -1); + checks.check("process alias rejects null diagnostic slot", yajl_json_process(parser.value, "{}", 2, nullptr) == -1); + { + ErrorSlot error; + checks.check("complete rejects null parser", yajl_json_complete(nullptr, &error.value) == -1); + checks.check("null parser complete clears diagnostic", error.replaced() && !error.value); + } + checks.check("complete rejects null diagnostic slot", yajl_json_complete(parser.value, nullptr) == -1); + { + ErrorSlot error; + checks.check("empty null buffer is a valid chunk", yajl_json_process_chunk(parser.value, nullptr, 0, &error.value) == 1); + checks.check("empty chunk clears diagnostic", error.replaced() && !error.value); + } + checks.success("parser usable after invalid calls", parse(parser.value, "null"), ": \n"); +} + +void split_case(Checks& checks, const std::string& name, const std::string& input, + bool valid, const std::string& expected = "") { + for (std::size_t split = 0; split <= input.size(); ++split) { + const std::string label = name + " split " + std::to_string(split); + Parser parser; + parser.value->silence = valid ? 0 : 1; + CapturedOutput output; + bool failed = false; + for (unsigned int part = 0; part < 2 && !failed; ++part) { + const std::size_t offset = part ? split : 0; + const std::size_t length = part ? input.size() - split : split; + ErrorSlot error; + // Exercise both public entry points with identical semantics. + const int rc = part ? yajl_json_process(parser.value, input.data() + offset, + static_cast(length), &error.value) : + yajl_json_process_chunk(parser.value, input.data() + offset, + static_cast(length), &error.value); + checks.check(label + ": chunk replaces old diagnostic", error.replaced()); + checks.check(label + ": chunk result and diagnostic agree", + (rc == 1 && !error.value) || (rc == -1 && error.value && *error.value)); + failed = rc != 1; + } + if (!failed) { + ErrorSlot error; + const int rc = yajl_json_complete(parser.value, &error.value); + checks.check(label + ": complete replaces old diagnostic", error.replaced()); + checks.check(label + ": complete result and diagnostic agree", + (rc == 1 && !error.value) || (rc == -1 && error.value && *error.value)); + failed = rc != 1; + } + checks.check(label + ": expected acceptance", failed != valid); + checks.check(label + ": byte-exact output", output.read() == (valid ? expected : "")); + if (valid) { + ErrorSlot complete_error; + checks.check(label + ": rejects duplicate completion", + yajl_json_complete(parser.value, &complete_error.value) == -1); + checks.check(label + ": duplicate completion has fresh diagnostic", + complete_error.replaced() && complete_error.value && *complete_error.value); + ErrorSlot chunk_error; + checks.check(label + ": rejects chunks after completion", + yajl_json_process_chunk(parser.value, " ", 1, &chunk_error.value) == -1); + checks.check(label + ": post-completion chunk has fresh diagnostic", + chunk_error.replaced() && chunk_error.value && *chunk_error.value); + checks.check(label + ": failed repeated calls do not emit more output", output.read() == expected); + } + } +} + +void chunk_boundaries(Checks& checks) { + split_case(checks, "empty object", "{}", true); + split_case(checks, "empty array", "[]", true); + split_case(checks, "root integer EOF", "18446744073709551616", true, ": 18446744073709551616\n"); + split_case(checks, "root exponent EOF", "-0.1200e+123", true, ": -0.1200e+123\n"); + split_case(checks, "raw multibyte UTF-8", "{\"\xc3\xa4\":\"\xe6\x97\xa5\xf0\x9f\x98\x80\"}", true, + "\xc3\xa4: \xe6\x97\xa5\xf0\x9f\x98\x80\n"); + split_case(checks, "surrogate pair", "{\"emoji\":\"\\uD83D\\uDE00\"}", true, "emoji: \xf0\x9f\x98\x80\n"); + split_case(checks, "escaped quote and slash", "{\"quote\":\"a\\\"b\\\\c\"}", true, "quote: a\"b\\c\n"); + const char nul_output[] = "a\0b: x\0y\n"; + split_case(checks, "decoded NUL", "{\"a\\u0000b\":\"x\\u0000y\"}", true, + std::string(nul_output, sizeof nul_output - 1)); + split_case(checks, "root and nested arrays", "[[null,true],{\"k\":false},2]", true, + "array.array.array: \narray.array.array: true\narray.array.k: false\narray.array: 2\n"); + split_case(checks, "dotted parents and siblings", "{\"a.b\":[{\"c.d\":1},2],\"after\":3}", true, + "a.b.a.b.c.d: 1\na.b.a.b: 2\nafter: 3\n"); + split_case(checks, "repeated and empty keys", "{\"\":{\"\":1},\"k\":2,\"k\":3}", true, + ": 1\nk: 2\nk: 3\n"); + + const std::vector> invalid = { + {"empty input", ""}, {"whitespace only", " \r\n\t"}, + {"truncated object", "{\"a\":1"}, {"truncated nested array", "[[true]"}, + {"truncated string", "\"text"}, {"truncated number", "1e+"}, + {"trailing document", "{}[]"}, {"trailing garbage", "null x"}, + {"trailing comma", "[1,]"}, {"missing separator", "[1 2]"}, + {"invalid escape", "\"\\q\""}, {"high surrogate alone", "\"\\ud800\""}, + {"low surrogate alone", "\"\\udc00\""}, + {"high surrogate plus ordinary unit", "\"\\ud800\\u0041\""}, + {"high surrogate plus high unit", "\"\\ud800\\ud800\""}, + {"surrogate plus raw continuation", "\"\\ud800a\""}, + {"overlong UTF-8", std::string("\"\xc0\xaf\"", 4)}, + {"UTF-8 surrogate code point", std::string("\"\xed\xa0\x80\"", 5)}, + {"UTF-8 beyond U+10FFFF", std::string("\"\xf4\x90\x80\x80\"", 6)}, + {"bare continuation", std::string("\"\x80\"", 3)}, + {"truncated UTF-8", std::string("\"\xe2\x82", 3)}, + {"raw NUL in string", std::string("\"a\0b\"", 5)}, + {"raw NUL after document", std::string("{}\0", 3)} + }; + for (const auto& test : invalid) split_case(checks, test.first, test.second, false); + + // Feed one byte per call as well as two chunks: lexer/guard state must span + // several calls, rather than merely preserving a final partial token. + Parser parser; + CapturedOutput output; + const std::string input = "{\"\xc3\xa4\":\"\\ud83d\\ude00\",\"n\":-0e+3}"; + for (std::size_t i = 0; i < input.size(); ++i) { + ErrorSlot error; + checks.check("byte-at-a-time chunk " + std::to_string(i), + yajl_json_process_chunk(parser.value, input.data() + i, 1, &error.value) == 1 && + error.replaced() && !error.value); + } + ErrorSlot error; + checks.check("byte-at-a-time completion", yajl_json_complete(parser.value, &error.value) == 1 && + error.replaced() && !error.value); + checks.check("byte-at-a-time byte-exact output", output.read() == "\xc3\xa4: \xf0\x9f\x98\x80\nn: -0e+3\n"); +} + +void limits_and_reuse(Checks& checks) { + Parser parser; + parser.value->silence = 1; + parser.value->depth_limit = 0; + for (const auto& value : {"null", "true", "false", "1", "\"x\""}) + checks.success(std::string("zero depth allows primitive ") + value, parse(parser.value, value)); + for (const auto& value : {"{}", "[]"}) + checks.failure(std::string("zero depth rejects container ") + value, parse(parser.value, value)); + parser.value->depth_limit = 3; + parser.value->arg_num_limit = 0; + for (const auto& value : {"{}", "[]", "{\"a\":[{},[]]}"}) + checks.success(std::string("zero arguments allows empty containers ") + value, parse(parser.value, value)); + for (const auto& value : {"null", "true", "false", "0", "\"\"", "{\"a\":null}", "[false]"}) + checks.failure(std::string("zero arguments rejects value ") + value, parse(parser.value, value)); + parser.value->arg_num_limit = 2; + checks.success("array exactly at depth limit", parse(parser.value, "[[[null]]]")); + checks.failure("array above depth limit", parse(parser.value, "[[[[null]]]]")); + checks.success("object exactly at depth limit", parse(parser.value, "{\"a\":{\"b\":{\"c\":true}}}")); + checks.failure("object above depth limit", parse(parser.value, "{\"a\":{\"b\":{\"c\":{}}}}")); + checks.success("bool and null exactly at argument limit", parse(parser.value, "{\"a\":null,\"b\":false}")); + checks.failure("bool and null above argument limit", parse(parser.value, "[null,true,false]")); + checks.success("empty containers consume no arguments", parse(parser.value, "{\"a\":{},\"b\":[],\"c\":null,\"d\":true}")); + for (const auto& value : {"null", "true", "false", "0", "\"\""}) { + checks.success(std::string("repeated scalar exactly at count: ") + value, + parse(parser.value, std::string("[") + value + "," + value + "]")); + checks.failure(std::string("repeated scalar above count: ") + value, + parse(parser.value, std::string("[") + value + "," + value + "," + value + "]")); + } + const Result incomplete = parse(parser.value, "{\"leaked\":{\"x\":true,\"y\":"); + checks.failure("nested truncated input", incomplete); + checks.check("silent malformed document has no argument output", incomplete.output.empty()); + checks.success("reuse after syntax failure", parse(parser.value, "{\"fresh\":true}")); + checks.check("reuse preserves caller settings", parser.value->depth_limit == 3 && + parser.value->arg_num_limit == 2 && parser.value->silence == 1); + parser.value->silence = 0; + checks.success("fresh path after failed file", parse(parser.value, "{\"fresh\":true}"), "fresh: true\n"); + checks.success("root array path", parse(parser.value, "[null,false]"), "array.array: \narray.array: false\n"); + checks.success("root scalar clears preceding array path", parse(parser.value, "2"), ": 2\n"); + const Result count_failure = parse(parser.value, "{\"a\":null,\"b\":true,\"c\":false}"); + checks.failure("retained count limit while unsilenced", count_failure); + checks.check("count failure emits only permitted arguments", count_failure.output == "a: \nb: true\n"); + checks.success("reuse after count failure", parse(parser.value, "{\"after\":false}"), "after: false\n"); + parser.value->depth_limit = 1; + checks.failure("retained tight depth limit", parse(parser.value, "{\"a\":[]}")); + checks.success("reuse after depth failure", parse(parser.value, "{\"after\":null}"), "after: \n"); + checks.failure("file reader finalizes root numeric syntax", parse(parser.value, "1e+")); + checks.success("reuse after EOF syntax error", parse(parser.value, "-0.0"), ": -0.0\n"); + checks.failure("file reader rejects trailing content", parse(parser.value, "{} true")); + checks.success("reuse after trailing content", parse(parser.value, "{}")); + parser.value->silence = 5; + checks.success("nonzero silence persists", parse(parser.value, "{\"quiet\":true}")); + checks.failure("silent mode still validates", parse(parser.value, "{\"quiet\":true,}")); + checks.success("silence persists after error", parse(parser.value, "{\"quiet\":false}")); + + // A file parse starts a fresh document even if callers previously used the + // incremental API and left a partially consumed or failed document behind. + { + ErrorSlot error; + checks.check("incremental parse rejected after finalized file", + yajl_json_process_chunk(parser.value, "{}", 2, &error.value) == -1 && + error.replaced() && error.value && *error.value); + } + checks.success("file parse resets finalized incremental state", parse(parser.value, "{}")); + { + Parser incremental; + incremental.value->silence = 1; + incremental.value->depth_limit = 2; + incremental.value->arg_num_limit = 1; + const char unfinished[] = "{\"stale\":["; + ErrorSlot chunk_error; + checks.check("unfinished incremental document initially accepted", + yajl_json_process_chunk(incremental.value, unfinished, sizeof unfinished - 1, + &chunk_error.value) == 1 && chunk_error.replaced() && !chunk_error.value); + ErrorSlot complete_error; + checks.check("unfinished incremental document rejected on completion", + yajl_json_complete(incremental.value, &complete_error.value) == -1 && + complete_error.replaced() && complete_error.value && *complete_error.value); + checks.success("file parse resets failed incremental document", + parse(incremental.value, "{\"fresh\":true}")); + checks.check("incremental-to-file reset preserves settings", + incremental.value->silence == 1 && incremental.value->depth_limit == 2 && + incremental.value->arg_num_limit == 1); + checks.failure("reset still enforces incremental caller count limit", + parse(incremental.value, "[null,true]")); + } +} + +void length_boundaries(Checks& checks) { + Parser parser; + const std::size_t limit = JSON_STRING_SIZE; + const std::string key(limit - 1, 'k'); + const std::string value(limit - 1, 'v'); + checks.success("key exactly below byte limit", parse(parser.value, "{\"" + key + "\":null}"), key + ": \n"); + checks.failure("key at byte limit", parse(parser.value, "{\"" + key + "k\":null}")); + checks.success("value exactly below byte limit", parse(parser.value, "{\"v\":\"" + value + "\"}"), "v: " + value + "\n"); + checks.failure("value at byte limit", parse(parser.value, "{\"v\":\"" + value + "v\"}")); + const std::string parent((limit - 2) / 2, 'p'); + const std::string child(limit - 2 - parent.size(), 'c'); + checks.success("full path exactly below byte limit", + parse(parser.value, "{\"" + parent + "\":{\"" + child + "\":1}}"), parent + "." + child + ": 1\n"); + checks.failure("full path at byte limit", parse(parser.value, + "{\"" + parent + "\":{\"" + child + "c\":1}}")); + checks.failure("oversized path entering empty container", parse(parser.value, + "{\"" + parent + "\":{\"" + child + "c\":{}}}")); + // A valid decoded token can span several file-reader chunks. Length limits + // apply to decoded bytes, not to the longer escape spelling in the input. + std::string escaped_key; + std::string escaped_value; + for (std::size_t i = 0; i < limit - 1; ++i) { + escaped_key += "\\u006b"; + escaped_value += "\\u0076"; + } + checks.success("escaped key below decoded byte limit across file chunks", + parse(parser.value, "{\"" + escaped_key + "\":null}"), key + ": \n"); + checks.failure("escaped key at decoded byte limit", + parse(parser.value, "{\"" + escaped_key + "\\u006b\":null}")); + checks.success("escaped value below decoded byte limit across file chunks", + parse(parser.value, "{\"v\":\"" + escaped_value + "\"}"), "v: " + value + "\n"); + checks.failure("escaped value at decoded byte limit", + parse(parser.value, "{\"v\":\"" + escaped_value + "\\u0076\"}")); + parser.value->silence = 1; + checks.failure("silent key length still enforced", parse(parser.value, "{\"" + key + "k\":0}")); + checks.failure("silent value length still enforced", parse(parser.value, "\"" + value + "v\"")); + checks.failure("oversized token guard", parse(parser.value, "\"" + std::string(limit * 8 + 32, 'x') + "\"")); + checks.failure("oversized number guard", parse(parser.value, std::string(limit * 8 + 32, '1'))); + checks.success("reuse after length errors", parse(parser.value, "{\"fresh\":true}")); + parser.value->silence = 0; + const char expected[] = "a\0b.c\0d: v\0x\na\0b.after: false\nlast: \n"; + checks.success("NUL parents restore with explicit lengths", parse(parser.value, + "{\"a\\u0000b\":{\"c\\u0000d\":\"v\\u0000x\",\"after\":false},\"last\":null}"), + std::string(expected, sizeof expected - 1)); + checks.success("dotted keys are not structural pop delimiters", parse(parser.value, + "{\"a.b\":{\"c.d\":1},\"after\":2}"), "a.b.c.d: 1\nafter: 2\n"); +} + +} // namespace + +int main() { + try { + Checks checks; + invalid_arguments(checks); + chunk_boundaries(checks); + limits_and_reuse(checks); + length_boundaries(checks); + return checks.finish(); + } catch (const std::exception& error) { + std::fprintf(stderr, "FAIL: YAJL C API test setup: %s\n", error.what()); + return 1; + } +} +#else +int main() { + std::fprintf(stderr, "YAJL C API tests require an enabled YAJL engine\n"); + return 77; +} +#endif From b4e1eb980fed135fb9195f4f1dbbb9aceed73d04 Mon Sep 17 00:00:00 2001 From: jens Steinbacher Date: Tue, 13 Oct 2026 13:56:05 +0000 Subject: [PATCH 03/18] Stream RapidJSON and nlohmann SAX adapters and enforce parser contracts --- docs/adapter-stack.md | 42 +++ src/nlparser.cpp | 462 +++++++++++++++-------------- src/rjparser.cpp | 496 ++++++++++++++++---------------- tests/benchmark_jsoncons.py | 5 +- tests/regression_tests.py | 4 + tests/sax_adapter_api_tests.cpp | 393 +++++++++++++++++++++++++ 6 files changed, 910 insertions(+), 492 deletions(-) create mode 100644 tests/sax_adapter_api_tests.cpp diff --git a/docs/adapter-stack.md b/docs/adapter-stack.md index 9304890..2bd41d3 100644 --- a/docs/adapter-stack.md +++ b/docs/adapter-stack.md @@ -52,6 +52,48 @@ g++ -std=c++17 -Wall -Wextra -Werror $CPPFLAGS \ /tmp/pathhandler-tests ``` +## 2. Streaming RapidJSON and nlohmann SAX adapters + +Both file entry points now stream bounded input instead of loading a fixed-size +whole-file buffer. RapidJSON uses the public FileReadStream and iterative Reader +with encoding validation and raw-number events; nlohmann uses its public SAX +stream API. Each parse gets fresh state, C entry points contain exceptions, and +checked setter values preserve configuration after invalid calls. Boolean/null +callbacks propagate argument-limit failures. Keys/values are length-aware, +including embedded NULs; output and file-read failures propagate. + +Narrow raw-NUL, Unicode and token-size guards address the native readers' EOF +sentinels and resource limits. No DOM is constructed. RapidJSON's unsafe Int64/ +Uint64 pointer casts are replaced with exact integer text. nlohmann's float +callback uses its provided original token, so delivered decimals and large +integer lexemes are not rounded through to_string(double). + +The installed RapidJSON 1.1 and nlohmann 3.11 parsers still reject extremely large +positive floating exponents before callbacks. Those required golden cases remain +failures with explicit range diagnostics. nlohmann's integer event normalizes +-0 to 0; it has an explicit expected spelling, matching the jsoncons integer +semantics. No output-wide numerical normalization is applied. + +C API zero limits have the same meaning as jsoncons/YAJL: zero argument limit +allows empty containers but no scalars; zero depth allows only root primitives. +The CLI continues to require positive limits. + +Fresh matrices: RapidJSON **228 passed, 4 failed, 0 skipped**; nlohmann **228 +passed, 4 failed, 0 skipped**. All four failures per engine are positive exponent +range cases in output/silent mode. The generic required assertions remain intact. +The new `tests/sax_adapter_api_tests.cpp` checks zero limits, invalid setters, +reuse, error ownership and byte-exact output directly across both C interfaces. +It passed **291 checks per engine (582 total)**, including ASan/UBSan with leak +detection. Either engine can be compiled alone; both disabled return skip code +77 rather than a successful test result. + +```sh +g++ -std=c++17 -Wall -Wextra -Werror $CPPFLAGS -I. \ + tests/sax_adapter_api_tests.cpp src/rjparser.cpp src/nlparser.cpp \ + $LDFLAGS -o /tmp/sax-api-tests +/tmp/sax-api-tests +``` + ## Optional JSON Schema validation No schema validator dependency is introduced. Schema rules are a separate diff --git a/src/nlparser.cpp b/src/nlparser.cpp index 4c1a685..a83deaa 100644 --- a/src/nlparser.cpp +++ b/src/nlparser.cpp @@ -1,302 +1,298 @@ #include "nlparser.h" -#include -#include -#include -#include -#include -#include - #if HAVE_NLOHMANNJSON #include "pathhandler.h" #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct nl_parser { + unsigned int max_depth = 500; + unsigned int max_args = 500; + bool silence = false; +}; + +namespace { + using nljson = nlohmann::json; -class NLSAXHandler : public nlohmann::json_sax{ - public: - explicit NLSAXHandler(): - m_max_depth(0), - m_current_depth(0), - m_depth_limit_exceeded(false), - m_arg_num_counter(0), - m_arg_num_limit(0), - m_arg_num_limit_exceeded(false), - m_silence(false), - m_path() - {}; - ~NLSAXHandler() {}; - - static bool init() { - return true; +// The public SAX API still materializes individual strings and number tokens. +// Guard their raw size before the lexer can allocate an unbounded token. Six +// source bytes per decoded byte allow legal \uXXXX escapes at the value limit. +// Reject raw NUL as well: nlohmann 3.11.3 otherwise treats it as end of input. +class CheckedInputBuffer final : public std::streambuf { + std::FILE* file_ = nullptr; + const char* memory_ = nullptr; + std::size_t remaining_ = 0; + std::array buffer_{}; + bool in_string_ = false; + bool escaped_ = false; + bool in_number_ = false; + std::size_t token_length_ = 0; + + static bool number_character(unsigned char c) { + return (c >= '0' && c <= '9') || c == '-' || c == '+' || + c == '.' || c == 'e' || c == 'E'; } - bool addArgument(std::string_view value) { - m_arg_num_counter++; - if (m_arg_num_limit > 0 && - m_arg_num_counter > m_arg_num_limit) { - m_arg_num_limit_exceeded = true; - std::cerr << "Argument number limit exceeded" << std::endl; - return false; - } - - if (m_silence) { - return true; + void check(unsigned char c) { + if (c == 0) throw std::invalid_argument("Invalid raw NUL in JSON input"); + if (in_string_) { + if (!escaped_ && c == '"') { + in_string_ = false; + token_length_ = 0; + return; + } + if (++token_length_ > static_cast(JSON_STRING_SIZE) * 6) + throw std::length_error("JSON string token too long"); + if (escaped_) escaped_ = false; + else if (c == '\\') escaped_ = true; + return; } - - char argname[JSON_STRING_SIZE]; - /* - * Argument name is 'm_prefix + m_current_key' - */ - m_path.buildArgName(argname); - if (m_path.m_error) { - std::cerr << "Argument name too long" << std::endl; - return false; + if (in_number_) { + if (number_character(c)) { + if (++token_length_ >= JSON_STRING_SIZE) + throw std::length_error("JSON number token too long"); + return; + } + in_number_ = false; + token_length_ = 0; } - if (value.size() >= JSON_STRING_SIZE) { - std::cerr << "Argument value too long" << std::endl; - return false; + if (c == '"') { + in_string_ = true; + escaped_ = false; + } else if ((c >= '0' && c <= '9') || c == '-') { + in_number_ = true; + token_length_ = 1; } - std::cout << argname << ": " << value << std::endl; - return true; } - /* NlohmannJSON mandatory methods */ - bool start_object(std::size_t elements) noexcept override { - m_path.push(); - if (m_path.m_error) { - std::cerr << "Argument name too long" << std::endl; - return false; - } - m_current_depth++; - if (m_current_depth > m_max_depth) { - m_depth_limit_exceeded = true; - return false; + int_type underflow() override { + std::size_t count; + if (file_) { + count = std::fread(buffer_.data(), 1, buffer_.size(), file_); + if (std::ferror(file_)) throw std::runtime_error("Error reading JSON file"); + } else { + count = remaining_ < buffer_.size() ? remaining_ : buffer_.size(); + if (count != 0) std::memcpy(buffer_.data(), memory_, count); + memory_ += count; + remaining_ -= count; } - return true; + if (count == 0) return traits_type::eof(); + for (std::size_t i = 0; i < count; ++i) + check(static_cast(buffer_[i])); + setg(buffer_.data(), buffer_.data(), buffer_.data() + count); + return traits_type::to_int_type(*gptr()); } - bool end_object() noexcept override { - m_path.pop(); - m_current_depth--; + public: + explicit CheckedInputBuffer(std::FILE* file) : file_(file) {} + CheckedInputBuffer(const char* memory, std::size_t length) + : memory_(memory), remaining_(length) {} +}; - return true; - } +class NLSAXHandler final : public nlohmann::json_sax { + const nl_parser& settings_; + unsigned int depth_ = 0; + unsigned int arguments_ = 0; + PathHandler path_; + std::string error_; - bool start_array(std::size_t elements) noexcept override { - m_path.startArraySpecial(); - if (m_path.m_error) { - std::cerr << "Argument name too long" << std::endl; - return false; - } - m_current_depth++; - if (m_current_depth > m_max_depth) { - m_depth_limit_exceeded = true; - return false; - } - return true; + bool fail(const char* message) { + error_ = message; + return false; } - bool end_array() noexcept override { - m_path.pop(); - m_current_depth--; + bool begin(bool array) { + if (depth_ >= settings_.max_depth) return fail("JSON depth limit exceeded"); + if (array) path_.startArraySpecial(); + else path_.push(); + if (path_.m_error) return fail("Argument name too long"); + ++depth_; return true; } - bool key(nljson::string_t& val) override { - m_path.setKey(val.c_str(), val.size()); - if (m_path.m_error) { - std::cerr << "Argument name too long" << std::endl; - return false; - } + bool end() { + if (depth_ == 0) return fail("Invalid JSON container state"); + path_.pop(); + --depth_; + if (path_.m_error) return fail("Invalid JSON argument path"); return true; } - bool string(nljson::string_t& v) override { - return addArgument(std::string_view(v)); - } - - bool number_integer(nljson::number_integer_t v) override { - std::string val = std::to_string(v); - return addArgument(std::string_view(val)); + bool argument(std::string_view value) { + if (arguments_ >= settings_.max_args) + return fail("Argument number limit exceeded"); + const std::string name = path_.name(); + if (path_.m_error) return fail("Argument name too long"); + if (value.size() >= JSON_STRING_SIZE) return fail("Argument value too long"); + // Unlimited argument mode must not wrap the counter on enormous input. + ++arguments_; + if (settings_.silence) return true; + if (std::fwrite(name.data(), 1, name.size(), stdout) != name.size() || + std::fwrite(": ", 1, 2, stdout) != 2 || + std::fwrite(value.data(), 1, value.size(), stdout) != value.size() || + std::fputc('\n', stdout) == EOF) + return fail("Unable to write JSON argument output"); + return true; } - bool number_unsigned(nljson::number_unsigned_t v) override { - std::string val = std::to_string(v); - return addArgument(std::string_view(val)); + public: + explicit NLSAXHandler(const nl_parser& settings) : settings_(settings) {} + const std::string& error() const { return error_; } + + bool start_object(std::size_t) override { return begin(false); } + bool end_object() override { return end(); } + bool start_array(std::size_t) override { return begin(true); } + bool end_array() override { return end(); } + bool key(nljson::string_t& value) override { + path_.setKey(value.data(), value.size()); + return !path_.m_error || fail("Argument name too long"); } - - bool number_float(nljson::number_float_t v, const nljson::string_t& s) override { - (void)s; - std::string val = std::to_string(v); - return addArgument(std::string_view(val)); + bool string(nljson::string_t& value) override { return argument(value); } + bool number_integer(nljson::number_integer_t value) override { + // nlohmann's integer event has no original token; this normalizes -0. + return argument(std::to_string(value)); } - - bool null() override { - addArgument(std::string_view("")); - return true; + bool number_unsigned(nljson::number_unsigned_t value) override { + return argument(std::to_string(value)); } - - bool boolean(bool b) override { - if (b) { - addArgument(std::string_view("true")); - } else { - addArgument(std::string_view("false")); - } - return true; + bool number_float(nljson::number_float_t, const nljson::string_t& token) override { + // Preserve the lexer spelling, including integers outside uint64/int64, + // instead of emitting the rounded double. Nonfinite overflow is rejected + // by nlohmann before this event and remains an explicit parse failure. + return argument(token); } - - bool parse_error(std::size_t, - const std::string&, - const nlohmann::detail::exception&) override { + bool null() override { return argument(""); } + bool boolean(bool value) override { return argument(value ? "true" : "false"); } + bool binary(nljson::binary_t&) override { return fail("Unexpected binary JSON event"); } + bool parse_error(std::size_t, const std::string&, + const nlohmann::detail::exception& error) override { + error_ = error.what(); return false; } +}; - bool binary(nlohmann::json::binary_t& val) override { - (void)val; // skip binary data - return true; - } - /* NlohmannJSON mandatory methods END */ - - void setMaxDepth(double max_depth) { - m_max_depth = max_depth; - } - - void setMaxArgNum(double max_arg_num) { - m_arg_num_limit = static_cast(max_arg_num); - } +int fail(char** error_msg, const char* message, int status = 2) { + if (error_msg) *error_msg = ::strdup(message); + return status; +} - void setSilence(int silence) { - if (silence) { - m_silence = true; - } else { - m_silence = false; - } - } +bool valid_limit(double value) { + return std::isfinite(value) && value >= 0 && std::floor(value) == value && + value <= std::numeric_limits::max(); +} - private: - double m_max_depth; - int64_t m_current_depth; - bool m_depth_limit_exceeded; - long int m_arg_num_counter; - long int m_arg_num_limit; - bool m_arg_num_limit_exceeded; - bool m_silence; - PathHandler m_path; +struct FileCloser { + void operator()(std::FILE* file) const noexcept { std::fclose(file); } }; -struct nl_parser { - NLSAXHandler* impl; -}; +int parse(nl_parser* parser, CheckedInputBuffer& source, char** error_msg) { + try { + // A fresh visitor keeps failed or completed documents from carrying + // depth, argument counts, errors or paths into a subsequent parse. + NLSAXHandler handler(*parser); + std::istream input(&source); + const bool ok = nljson::sax_parse(input, &handler, + nljson::input_format_t::json, true, false); + if (!ok) return fail(error_msg, handler.error().empty() ? + "NlohmannJSON parsing failed" : handler.error().c_str()); + if (!parser->silence && std::fflush(stdout) != 0) + return fail(error_msg, "Unable to write JSON argument output"); + return 0; + } catch (const std::exception& error) { + return fail(error_msg, error.what()); + } catch (...) { + return fail(error_msg, "Unexpected NlohmannJSON parser failure"); + } +} -extern "C" int nl_json_init(nl_parser **parser, char **error_msg) { - assert(error_msg != NULL); - assert(parser != NULL); +} // namespace +extern "C" int nl_json_init(nl_parser** parser, char** error_msg) { + if (error_msg) *error_msg = nullptr; + if (parser) *parser = nullptr; + if (!parser || !error_msg) return -1; try { - *parser = new nl_parser{}; - } - catch (const std::exception &e) { - *parser = nullptr; - *error_msg = strdup(e.what()); - return 1; + *parser = new nl_parser; + return 0; + } catch (const std::exception& error) { + return fail(error_msg, error.what(), -1); + } catch (...) { + return fail(error_msg, "Unable to initialize NlohmannJSON parser", -1); } - - *error_msg = NULL; - (*parser)->impl = new NLSAXHandler(); - (*parser)->impl->init(); - return 0; } -/** - * Frees the resources used for JSON parsing. - */ -extern "C" int nl_json_cleanup(nl_parser *parser) { - delete(parser->impl); - delete(parser); +extern "C" int nl_json_cleanup(nl_parser* parser) { + delete parser; return 0; } -extern "C" int nl_parse_buffer(nl_parser *parser, const char *buf, unsigned int len, - char **error_msg) { - if (!buf || !error_msg) return -1; - - - bool ok = nlohmann::json::sax_parse(buf, buf + len, parser->impl); - - if (!ok) { - fprintf(stderr, "NlohmannJSON error: parsing failed\n"); - return 2; +extern "C" int nl_parse_buffer(nl_parser* parser, const char* buffer, + unsigned int length, char** error_msg) { + if (error_msg) *error_msg = nullptr; + if (!parser || !buffer || !error_msg) + return fail(error_msg, "Invalid NlohmannJSON parser arguments", -1); + try { + CheckedInputBuffer source(buffer, length); + return parse(parser, source, error_msg); + } catch (const std::exception& error) { + return fail(error_msg, error.what()); + } catch (...) { + return fail(error_msg, "Unexpected NlohmannJSON parser failure"); } - return 0; } -extern "C" int nl_set_max_depth(nl_parser *parser, double max_depth) { - parser->impl->setMaxDepth(max_depth); - return 0; +extern "C" int nl_parse_file(nl_parser* parser, const char* filename, char** error_msg) { + if (error_msg) *error_msg = nullptr; + if (!parser || !filename || !error_msg) + return fail(error_msg, "Invalid NlohmannJSON parser arguments", -1); + try { + std::unique_ptr file(std::fopen(filename, "rb")); + if (!file) return fail(error_msg, "Unable to open JSON file", -1); + CheckedInputBuffer source(file.get()); + return parse(parser, source, error_msg); + } catch (const std::exception& error) { + return fail(error_msg, error.what()); + } catch (...) { + return fail(error_msg, "Unexpected NlohmannJSON parser failure"); + } } -extern "C" int nl_set_max_arg_num(nl_parser *parser, double max_arg_num) { - parser->impl->setMaxArgNum(max_arg_num); +extern "C" int nl_set_max_depth(nl_parser* parser, double value) { + if (!parser || !valid_limit(value)) return -1; + parser->max_depth = static_cast(value); return 0; } -extern "C" int nl_set_silence(nl_parser *parser, int silence) { - parser->impl->setSilence(silence); +extern "C" int nl_set_max_arg_num(nl_parser* parser, double value) { + if (!parser || !valid_limit(value)) return -1; + parser->max_args = static_cast(value); return 0; } -extern "C" int nl_parser_cleanup(nl_parser *parser) { - parser->impl->~NLSAXHandler(); +extern "C" int nl_set_silence(nl_parser* parser, int silence) { + if (!parser) return -1; + parser->silence = silence != 0; return 0; } -#ifndef FILE_BUFFER_SIZE -#define FILE_BUFFER_SIZE 10485760 -#endif - -extern "C" int nl_parse_file(nl_parser *parser, const char *filename, char **error_msg) { - if (!parser || !filename || !error_msg) return -1; - - FILE *fp = fopen(filename, "r"); - if (!fp) { - *error_msg = strdup("Unable to open file"); - return -1; - } - - std::vector buf(FILE_BUFFER_SIZE); - size_t len = fread(buf.data(), 1, FILE_BUFFER_SIZE - 1, fp); - int read_error = ferror(fp); - int too_long = 0; - if (!read_error && len == FILE_BUFFER_SIZE - 1) { - char extra; - too_long = fread(&extra, 1, 1, fp) == 1; - read_error = ferror(fp); - } - fclose(fp); - - if (read_error) { - *error_msg = strdup("Error reading file"); - return -1; - } - if (too_long) { - *error_msg = strdup("File too long"); - return -1; - } - if (len == 0) { - *error_msg = strdup("Zero bytes read from file"); - return -1; - } - buf[len] = '\0'; - - bool ok = nlohmann::json::sax_parse(buf.data(), buf.data() + len, parser->impl); - if (!ok) { - fprintf(stderr, "NlohmannJSON error: parsing failed\n"); - return 2; - } - return 0; +// Retain this old, undeclared symbol without manually destroying an object +// that nl_json_cleanup would subsequently destroy a second time. +extern "C" int nl_parser_cleanup(nl_parser* parser) { + return parser ? 0 : -1; } #endif diff --git a/src/rjparser.cpp b/src/rjparser.cpp index b6b21b4..6e72fd3 100644 --- a/src/rjparser.cpp +++ b/src/rjparser.cpp @@ -1,313 +1,299 @@ #include "rjparser.h" -#include -#include -#include -#include -#include - #if HAVE_RAPIDJSON #include "pathhandler.h" -#include "rapidjson/reader.h" -#include "rapidjson/filereadstream.h" -#include "rapidjson/stringbuffer.h" -#include "rapidjson/error/en.h" - -class RJSAXHandler { - public: - explicit RJSAXHandler(): - m_max_depth(0), - m_current_depth(0), - m_depth_limit_exceeded(false), - m_arg_num_counter(0), - m_arg_num_limit(0), - m_arg_num_limit_exceeded(false), - m_silence(false), - m_path() - {}; - ~RJSAXHandler() {}; +#include +#include +#include +#include +#include +#include +#include +#include - static bool init() { - return true; - } +#include +#include +#include +#include - bool addArgument(const std::string& value) { +struct rj_parser { + unsigned int max_depth = 500; + unsigned int max_args = 500; + bool silence = false; +}; - m_arg_num_counter++; - if (m_arg_num_limit > 0 && - m_arg_num_counter > m_arg_num_limit) { - m_arg_num_limit_exceeded = true; - std::cerr << "Argument number limit exceeded" << std::endl; - return false; - } +namespace { - if (m_silence) { - return true; - } - /* - * Argument name is 'm_prefix + m_current_key' - */ - char argname[JSON_STRING_SIZE]; - m_path.buildArgName(argname); - if (m_path.m_error) { - std::cerr << "Argument name too long" << std::endl; - return false; - } - if (value.size() >= JSON_STRING_SIZE) { - std::cerr << "Argument value too long" << std::endl; - return false; - } - std::cout << argname << ": " << value << std::endl; - return true; - } - - /* RapidJSON mandatory methods */ - bool StartObject() { // cppcheck-suppress unusedFunction - m_path.push(); - if (m_path.m_error) { - std::cerr << "Argument name too long" << std::endl; - return false; - } - m_current_depth++; - if (m_current_depth > m_max_depth) { - m_depth_limit_exceeded = true; - return false; +// RapidJSON keeps one decoded token and its iterative structural stack here. +// A checked allocator stops hostile unterminated strings/numbers from growing +// document-sized storage before a SAX callback can enforce the token limit. +class TemporaryAllocator { + public: + static const bool kNeedFree = true; + void* Malloc(std::size_t size) { return Realloc(nullptr, 0, size); } + void* Realloc(void* data, std::size_t, std::size_t size) { + constexpr std::size_t budget = + static_cast(JSON_STRING_SIZE) * 4 + 16384; + if (size > budget) + throw std::length_error("RapidJSON token/state allocation limit exceeded"); + if (size == 0) { + std::free(data); + return nullptr; } - return true; - } - - bool EndObject(rapidjson::SizeType) { // cppcheck-suppress unusedFunction - m_path.pop(); - m_current_depth--; - return true; + void* result = std::realloc(data, size); + if (result == nullptr) throw std::bad_alloc(); + return result; } + static void Free(void* data) { std::free(data); } +}; - bool StartArray() { // cppcheck-suppress unusedFunction - m_path.startArraySpecial(); - if (m_path.m_error) { - std::cerr << "Argument name too long" << std::endl; - return false; - } - m_current_depth++; - if (m_current_depth > m_max_depth) { - m_depth_limit_exceeded = true; - return false; +// RapidJSON uses zero both for its synthetic EOF sentinel and for an input +// NUL. A real byte advances Tell() when consumed; the EOF sentinel does not. +// Reject real NUL bytes rather than accepting a prematurely terminated JSON. +// FileReadStream retains its fixed-size fread buffer; no whole-file copy. +template +class CheckedStream { + Source& source_; + public: + using Ch = char; + explicit CheckedStream(Source& source) : source_(source) {} + Ch Peek() { + const Ch value = source_.Peek(); + if (value == '\0') { + const std::size_t offset = source_.Tell(); + source_.Take(); + if (source_.Tell() != offset) + throw std::invalid_argument("Unexpected NUL byte at offset " + + std::to_string(offset)); } - return true; + return value; } - - bool EndArray(rapidjson::SizeType) { // cppcheck-suppress unusedFunction - m_path.pop(); - m_current_depth--; - return true; + Ch Take() { + const Ch value = Peek(); + if (value != '\0') source_.Take(); + return value; } + std::size_t Tell() const { return source_.Tell(); } + Ch* PutBegin() { throw std::logic_error("In-situ parsing is disabled"); } + void Put(Ch) { throw std::logic_error("In-situ parsing is disabled"); } + void Flush() {} + std::size_t PutEnd(Ch*) { throw std::logic_error("In-situ parsing is disabled"); } +}; - bool Key(const char* k, size_t l, bool) { // cppcheck-suppress unusedFunction - m_path.setKey(k, l); - if (m_path.m_error) { - std::cerr << "Argument name too long" << std::endl; - return false; - } - return true; - } +struct FileCloser { + void operator()(std::FILE* file) const noexcept { std::fclose(file); } +}; - bool String(const char* v, size_t l, bool b) { // cppcheck-suppress unusedFunction - std::string val = std::string(reinterpret_cast(v), l); - return addArgument(val); - } +struct DiscardStream { + using Ch = char; + void Put(char) {} +}; - bool Int(int64_t v) { // cppcheck-suppress unusedFunction - std::string val = std::to_string(v); - return addArgument(val); +// kParseValidateEncodingFlag validates raw UTF-8, but RapidJSON 1.1 also +// encodes a lone escaped low surrogate. Validate decoded keys/values too. +bool valid_utf8(const char* value, std::size_t length) { + rapidjson::MemoryStream input(value, length); + DiscardStream output; + while (input.Tell() < length) { + if (!rapidjson::UTF8<>::Validate(input, output)) return false; } + return true; +} - bool Uint(uint64_t v) { // cppcheck-suppress unusedFunction - std::string val = std::to_string(v); - return addArgument(val); +class RJSAXHandler { + const rj_parser& settings_; + PathHandler path_; + std::size_t depth_ = 0; + std::size_t arguments_ = 0; + const char* error_ = nullptr; + + bool fail(const char* message) { + if (error_ == nullptr) error_ = message; + return false; } - - bool Double(double v) { // cppcheck-suppress unusedFunction - std::string val = std::to_string(v); - return addArgument(val); + bool check_path() { + return !path_.m_error || fail("Argument name too long"); } - - bool Bool(bool b) { // cppcheck-suppress unusedFunction - if (b) { - addArgument("true"); - } else { - addArgument("false"); - } - return true; + bool begin(bool array) { + if (depth_ >= settings_.max_depth) return fail("JSON depth limit exceeded"); + if (array) path_.startArraySpecial(); + else path_.push(); + ++depth_; + return check_path(); } - - bool Null() { // cppcheck-suppress unusedFunction - addArgument(""); - return true; + bool end() { + path_.pop(); + --depth_; + return check_path(); } - - bool RawNumber(const char* str, rapidjson::SizeType length, bool) { // cppcheck-suppress unusedFunction - std::string val = std::string(reinterpret_cast(str), length); - return addArgument(val); + bool argument(const char* value, std::size_t length) { + const std::string name = path_.name(); + if (!check_path()) return false; + if (length >= JSON_STRING_SIZE) return fail("Argument value too long"); + if (arguments_ >= settings_.max_args) + return fail("Argument number limit exceeded"); + if (arguments_ == std::numeric_limits::max()) + return fail("Argument number counter overflow"); + ++arguments_; + if (settings_.silence) return true; + const bool ok = std::fwrite(name.data(), 1, name.size(), stdout) == name.size() && + std::fwrite(": ", 1, 2, stdout) == 2 && + std::fwrite(value, 1, length, stdout) == length && + std::fputc('\n', stdout) != EOF; + return ok || fail("Unable to write JSON argument output"); } + bool integer(const std::string& value) { return argument(value.data(), value.size()); } - bool Int64(int64_t i) { // cppcheck-suppress unusedFunction - std::string val = std::string(reinterpret_cast(i)); - return addArgument(val); + public: + explicit RJSAXHandler(const rj_parser& settings) : settings_(settings) {} + const char* error() const { return error_; } + bool StartObject() { return begin(false); } + bool EndObject(rapidjson::SizeType) { return end(); } + bool StartArray() { return begin(true); } + bool EndArray(rapidjson::SizeType) { return end(); } + bool Key(const char* value, rapidjson::SizeType length, bool) { + if (length >= JSON_STRING_SIZE) return fail("Argument name too long"); + if (!valid_utf8(value, length)) return fail("Invalid Unicode in JSON key"); + path_.setKey(value, length); + return check_path(); } - - bool Uint64(uint64_t u) { // cppcheck-suppress unusedFunction - std::string val = std::string(reinterpret_cast(u)); - return addArgument(val); + bool String(const char* value, rapidjson::SizeType length, bool) { + if (length >= JSON_STRING_SIZE) return fail("Argument value too long"); + if (!valid_utf8(value, length)) return fail("Invalid Unicode in JSON string"); + return argument(value, length); } - - /* RapidJSON mandatory methods END */ - - void setMaxDepth(double max_depth) { - m_max_depth = max_depth; + bool RawNumber(const char* value, rapidjson::SizeType length, bool) { + // Use the original lexeme: no double conversion or integer casts. + // RapidJSON 1.1 still rejects numeric overflow before this callback. + return argument(value, length); } + bool Int(int value) { return integer(std::to_string(value)); } + bool Uint(unsigned int value) { return integer(std::to_string(value)); } + bool Int64(int64_t value) { return integer(std::to_string(value)); } + bool Uint64(uint64_t value) { return integer(std::to_string(value)); } + bool Double(double) { return fail("Unexpected non-lossless JSON number event"); } + bool Bool(bool value) { return argument(value ? "true" : "false", value ? 4 : 5); } + bool Null() { return argument("", 0); } +}; - void setMaxArgNum(double max_arg_num) { - m_arg_num_limit = static_cast(max_arg_num); +template +int parse(rj_parser* parser, Source& source, char** error_msg) { + RJSAXHandler handler(*parser); // Fresh counters/path, retained settings. + CheckedStream input(source); + rapidjson::GenericReader, rapidjson::UTF8<>, + TemporaryAllocator> reader; + constexpr unsigned flags = rapidjson::kParseNumbersAsStringsFlag | + rapidjson::kParseValidateEncodingFlag | rapidjson::kParseIterativeFlag; + const rapidjson::ParseResult result = reader.Parse(input, handler); + if (!result) { + std::string message = handler.error() ? handler.error() : + rapidjson::GetParseError_En(result.Code()); + message += " at offset " + std::to_string(result.Offset()); + *error_msg = strdup(message.c_str()); + return 2; } - - void setSilence(int silence) { - if (silence) { - m_silence = true; - } else { - m_silence = false; - } + if (!parser->silence && std::fflush(stdout) != 0) { + *error_msg = strdup("Unable to write JSON argument output"); + return 2; } + return 0; +} - private: - double m_max_depth; - int64_t m_current_depth; - bool m_depth_limit_exceeded; - long int m_arg_num_counter; - long int m_arg_num_limit; - bool m_arg_num_limit_exceeded; - bool m_silence; - PathHandler m_path; - -}; - -struct rj_parser { - RJSAXHandler impl; -}; +int set_limit(rj_parser* parser, double value, unsigned int* target) { + if (parser == nullptr || !std::isfinite(value) || value < 0 || + value > std::numeric_limits::max() || std::floor(value) != value) + return -1; + *target = static_cast(value); + return 0; +} -extern "C" int rj_json_init(rj_parser **parser, char **error_msg) { - assert(error_msg != NULL); - assert(parser != NULL); +} // namespace +extern "C" int rj_json_init(rj_parser** parser, char** error_msg) { + if (error_msg == nullptr) return -1; + *error_msg = nullptr; + if (parser == nullptr) return -1; + *parser = nullptr; try { *parser = new rj_parser{}; - } - catch (const std::exception &e) { - *parser = nullptr; + return 0; + } catch (const std::exception& e) { *error_msg = strdup(e.what()); return 1; + } catch (...) { + *error_msg = strdup("Unable to initialize RapidJSON parser"); + return 1; } - - *error_msg = NULL; - (*parser)->impl.init(); - return 0; } -/** - * Frees the resources used for JSON parsing. - */ -extern "C" int rj_json_cleanup(rj_parser *parser) { - delete(parser); - return 0; +extern "C" int rj_parse_buffer(rj_parser* parser, const char* buf, + unsigned int len, char** error_msg) { + if (error_msg == nullptr) return -1; + *error_msg = nullptr; + if (parser == nullptr || buf == nullptr) return -1; + try { + rapidjson::MemoryStream input(buf, len); + return parse(parser, input, error_msg); + } catch (const std::exception& e) { + *error_msg = strdup(e.what()); + return 2; + } catch (...) { + *error_msg = strdup("Unexpected RapidJSON parser failure"); + return 2; + } } -extern "C" int rj_parse_buffer(rj_parser *parser, const char *buf, unsigned int len, - char **error_msg) { - if (!buf || !error_msg) return -1; - - std::vector tmp(buf, buf + len); - tmp.push_back('\0'); - - rapidjson::StringStream ss(tmp.data()); - rapidjson::Reader reader; - - rapidjson::ParseResult pr = reader.Parse(ss, (parser)->impl); - if (!pr) { - fprintf(stderr, "RapidJSON error: %s at %zu\n", - rapidjson::GetParseError_En(pr.Code()), pr.Offset()); +extern "C" int rj_parse_file(rj_parser* parser, const char* filename, + char** error_msg) { + if (error_msg == nullptr) return -1; + *error_msg = nullptr; + if (parser == nullptr || filename == nullptr) return -1; + try { + std::unique_ptr file(std::fopen(filename, "rb")); + if (!file) { + *error_msg = strdup("Unable to open JSON file"); + return 2; + } + char buffer[16384]; + rapidjson::FileReadStream input(file.get(), buffer, sizeof(buffer)); + const int result = parse(parser, input, error_msg); + if (std::ferror(file.get())) { + std::free(*error_msg); + *error_msg = strdup("Error reading JSON file"); + return 2; + } + return result; + } catch (const std::exception& e) { + *error_msg = strdup(e.what()); + return 2; + } catch (...) { + *error_msg = strdup("Unexpected RapidJSON parser failure"); return 2; } - return 0; } -extern "C" int rj_set_max_depth(rj_parser *parser, double max_depth) { - parser->impl.setMaxDepth(max_depth); - return 0; +extern "C" int rj_set_max_depth(rj_parser* parser, double value) { + return set_limit(parser, value, parser ? &parser->max_depth : nullptr); } - -extern "C" int rj_set_max_arg_num(rj_parser *parser, double max_arg_num) { - parser->impl.setMaxArgNum(max_arg_num); - return 0; +extern "C" int rj_set_max_arg_num(rj_parser* parser, double value) { + return set_limit(parser, value, parser ? &parser->max_args : nullptr); } - -extern "C" int rj_set_silence(rj_parser *parser, int silence) { - parser->impl.setSilence(silence); +extern "C" int rj_set_silence(rj_parser* parser, int silence) { + if (parser == nullptr) return -1; + parser->silence = silence != 0; return 0; } - -extern "C" int rj_parser_cleanup(rj_parser *parser) { - parser->impl.~RJSAXHandler(); +extern "C" int rj_json_cleanup(rj_parser* parser) { + delete parser; return 0; } -#ifndef FILE_BUFFER_SIZE -#define FILE_BUFFER_SIZE 10485760 -#endif - -extern "C" int rj_parse_file(rj_parser *parser, const char *filename, char **error_msg) { - if (!parser || !filename || !error_msg) return -1; - - FILE *fp = fopen(filename, "r"); - if (!fp) { - *error_msg = strdup("Unable to open file"); - return -1; - } - - std::vector buf(FILE_BUFFER_SIZE); - size_t len = fread(buf.data(), 1, FILE_BUFFER_SIZE - 1, fp); - int read_error = ferror(fp); - int too_long = 0; - if (!read_error && len == FILE_BUFFER_SIZE - 1) { - char extra; - too_long = fread(&extra, 1, 1, fp) == 1; - read_error = ferror(fp); - } - fclose(fp); - - if (read_error) { - *error_msg = strdup("Error reading file"); - return -1; - } - if (too_long) { - *error_msg = strdup("File too long"); - return -1; - } - if (len == 0) { - *error_msg = strdup("Zero bytes read from file"); - return -1; - } - buf[len] = '\0'; - - rapidjson::StringStream ss(buf.data()); - rapidjson::Reader reader; - rapidjson::ParseResult pr = reader.Parse(ss, parser->impl); - if (!pr) { - fprintf(stderr, "RapidJSON error: %s at %zu\n", - rapidjson::GetParseError_En(pr.Code()), pr.Offset()); - return 2; - } - return 0; +// Preserve the legacy undeclared symbol; destroying the handler manually +// made a later rj_json_cleanup double-destroy its strings and containers. +extern "C" int rj_parser_cleanup(rj_parser* parser) { + return parser ? 0 : -1; } -#endif \ No newline at end of file +#endif diff --git a/tests/benchmark_jsoncons.py b/tests/benchmark_jsoncons.py index 9815ec3..64ae20f 100644 --- a/tests/benchmark_jsoncons.py +++ b/tests/benchmark_jsoncons.py @@ -207,7 +207,7 @@ def main(): "timing": "CLI parser elapsed seconds; independent fresh processes after warmups", "rss": "Linux wait4 child peak RSS in KiB via a small compiled C launcher", "file_buffer_size": file_buffer_size, - "yajl_maximum_file_bytes": file_buffer_size - 1, + "yajl_input": "attempt every requested size; streaming adapters have no FILE_BUFFER_SIZE file cap", "compiler": compiler, "caveat": "Original DOM adapter only parses/discards a DOM. SAX also builds argument paths and enforces argument/size limits. Synthetic cached-file workload; no ModSecurity performance conclusion."}, "inputs": [], "runs": []} @@ -236,9 +236,6 @@ def main(): if "YAJL" not in engines: report["runs"].append({"variant": "yajl", "input_bytes": info["bytes"], "status": "skipped", "reason": "YAJL is unavailable in the configured binary"}) - elif info["bytes"] > file_buffer_size - 1: - report["runs"].append({"variant": "yajl", "input_bytes": info["bytes"], - "status": "skipped", "reason": f"input exceeds configured YAJL file limit of {file_buffer_size - 1} bytes"}) else: variants.append(("yajl", binary, "YAJL")) diff --git a/tests/regression_tests.py b/tests/regression_tests.py index 8ea5dcd..278a9b9 100755 --- a/tests/regression_tests.py +++ b/tests/regression_tests.py @@ -33,6 +33,10 @@ class Case: def output_for(self, engine): # YAJL preserves -0 and exponent '+'. jsoncons' public lossless API # normalizes those spellings; other engines still must preserve value. + # nlohmann's native integer SAX event also normalizes integer -0. + # This is an explicit expected spelling, not output normalization. + if self.name == "negative-zero" and engine == "NLOHMANNJSON": + return self.expected if self.numeric_difference and engine != "JSONCONS": return self.yajl_observed return self.expected diff --git a/tests/sax_adapter_api_tests.cpp b/tests/sax_adapter_api_tests.cpp new file mode 100644 index 0000000..8363325 --- /dev/null +++ b/tests/sax_adapter_api_tests.cpp @@ -0,0 +1,393 @@ +// The CLI only accepts positive integral limits. Exercise both SAX adapters' +// public C APIs for zero limits, rejected setters, reuse, and binary-safe I/O. +#include "../src/rjparser.h" +#include "../src/nlparser.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if HAVE_RAPIDJSON || HAVE_NLOHMANNJSON +namespace { + +class TemporaryInput { + std::string path_; + public: + explicit TemporaryInput(const std::string& text) { + char name[] = "/tmp/jsonbench-sax-api-XXXXXX"; + const int fd = mkstemp(name); + if (fd < 0) throw std::runtime_error("mkstemp failed"); + path_ = name; + std::size_t offset = 0; + while (offset < text.size()) { + const ssize_t count = write(fd, text.data() + offset, text.size() - offset); + if (count < 0 && errno == EINTR) continue; + if (count <= 0) { + close(fd); + unlink(path_.c_str()); + throw std::runtime_error("temporary input write failed"); + } + offset += static_cast(count); + } + if (close(fd) != 0) { + unlink(path_.c_str()); + throw std::runtime_error("temporary input close failed"); + } + } + ~TemporaryInput() { unlink(path_.c_str()); } + TemporaryInput(const TemporaryInput&) = delete; + TemporaryInput& operator=(const TemporaryInput&) = delete; + const char* path() const { return path_.c_str(); } +}; + +class CapturedOutput { + FILE* file_ = nullptr; + int saved_fd_ = -1; + public: + CapturedOutput() { + if (std::fflush(stdout) != 0) throw std::runtime_error("stdout flush failed"); + file_ = std::tmpfile(); + if (!file_) throw std::runtime_error("stdout temporary file failed"); + saved_fd_ = dup(STDOUT_FILENO); + if (saved_fd_ < 0 || dup2(fileno(file_), STDOUT_FILENO) < 0) { + if (saved_fd_ >= 0) close(saved_fd_); + std::fclose(file_); + throw std::runtime_error("stdout capture failed"); + } + } + ~CapturedOutput() { + std::fflush(stdout); + dup2(saved_fd_, STDOUT_FILENO); + close(saved_fd_); + std::fclose(file_); + } + CapturedOutput(const CapturedOutput&) = delete; + CapturedOutput& operator=(const CapturedOutput&) = delete; + std::string read() { + if (std::fflush(stdout) != 0 || std::fseek(file_, 0, SEEK_SET) != 0) + throw std::runtime_error("captured output rewind failed"); + std::string result; + char buffer[1024]; + std::size_t length; + while ((length = std::fread(buffer, 1, sizeof buffer, file_)) != 0) + result.append(buffer, length); + if (std::ferror(file_)) throw std::runtime_error("captured output read failed"); + return result; + } +}; + +// The API replaces the slot and returns free()-owned diagnostics. It must not +// free the caller's previous allocation, including on invalid-argument paths. +class ErrorSlot { + char* original_; + public: + char* value; + ErrorSlot() : original_(static_cast(std::malloc(1))), value(original_) { + if (!original_) throw std::bad_alloc(); + *original_ = '\0'; + } + ~ErrorSlot() { + if (value != original_) std::free(value); + std::free(original_); + } + ErrorSlot(const ErrorSlot&) = delete; + ErrorSlot& operator=(const ErrorSlot&) = delete; + bool replaced() const { return value != original_; } +}; + +#if HAVE_RAPIDJSON +struct RapidJsonApi { + using Handle = rj_parser; + static const char* name() { return "RapidJSON"; } + static int init(Handle** p, char** e) { return rj_json_init(p, e); } + static int buffer(Handle* p, const char* b, unsigned int n, char** e) { return rj_parse_buffer(p, b, n, e); } + static int file(Handle* p, const char* f, char** e) { return rj_parse_file(p, f, e); } + static int depth(Handle* p, double n) { return rj_set_max_depth(p, n); } + static int arguments(Handle* p, double n) { return rj_set_max_arg_num(p, n); } + static int silence(Handle* p, int n) { return rj_set_silence(p, n); } + static int cleanup(Handle* p) { return rj_json_cleanup(p); } +}; +#endif + +#if HAVE_NLOHMANNJSON +struct NlohmannApi { + using Handle = nl_parser; + static const char* name() { return "NlohmannJSON"; } + static int init(Handle** p, char** e) { return nl_json_init(p, e); } + static int buffer(Handle* p, const char* b, unsigned int n, char** e) { return nl_parse_buffer(p, b, n, e); } + static int file(Handle* p, const char* f, char** e) { return nl_parse_file(p, f, e); } + static int depth(Handle* p, double n) { return nl_set_max_depth(p, n); } + static int arguments(Handle* p, double n) { return nl_set_max_arg_num(p, n); } + static int silence(Handle* p, int n) { return nl_set_silence(p, n); } + static int cleanup(Handle* p) { return nl_json_cleanup(p); } +}; +#endif + +template +class Parser { + public: + typename Api::Handle* value = nullptr; + Parser() { + ErrorSlot error; + if (Api::init(&value, &error.value) != 0 || !value || error.value) { + Api::cleanup(value); + throw std::runtime_error(std::string(Api::name()) + " initialization failed"); + } + } + ~Parser() { Api::cleanup(value); } + Parser(const Parser&) = delete; + Parser& operator=(const Parser&) = delete; +}; + +struct Result { + int status; + bool replaced; + bool null_error; + std::string error; + std::string output; +}; + +template +Result parse_path(typename Api::Handle* parser, const char* path) { + ErrorSlot error; + CapturedOutput output; + const int status = Api::file(parser, path, &error.value); + return {status, error.replaced(), !error.value, error.value ? error.value : "", output.read()}; +} + +template +Result parse(typename Api::Handle* parser, const std::string& text, bool file = false) { + if (file) { + TemporaryInput input(text); + return parse_path(parser, input.path()); + } + ErrorSlot error; + CapturedOutput output; + const int status = Api::buffer(parser, text.data(), static_cast(text.size()), &error.value); + return {status, error.replaced(), !error.value, error.value ? error.value : "", output.read()}; +} + +class Checks { + unsigned int total_ = 0; + unsigned int failures_ = 0; + public: + std::string engine; + void check(const std::string& name, bool passed) { + ++total_; + if (!passed) { + ++failures_; + std::fprintf(stderr, "FAIL: %s: %s\n", engine.c_str(), name.c_str()); + } + } + void success(const std::string& name, const Result& result, const std::string& output = "") { + check(name + ": succeeds", result.status == 0); + check(name + ": clears old diagnostic", result.replaced && result.null_error); + check(name + ": byte-exact output", result.output == output); + if (result.status != 0) + std::fprintf(stderr, " diagnostic: %s\n", result.error.c_str()); + } + void failure(const std::string& name, const Result& result) { + check(name + ": fails", result.status != 0); + check(name + ": fresh malloc-owned diagnostic", result.replaced && !result.null_error && !result.error.empty()); + } + unsigned int total() const { return total_; } + unsigned int failures() const { return failures_; } + int finish() const { + std::printf("SAX adapter C APIs: %u checks passed, %u failed\n", total_ - failures_, failures_); + return failures_ ? 1 : 0; + } +}; + +template +void invalid_arguments(Checks& checks) { + { + ErrorSlot error; + checks.check("init rejects null parser slot", Api::init(nullptr, &error.value) != 0); + checks.check("invalid init clears old diagnostic", error.replaced() && !error.value); + } + typename Api::Handle* handle = nullptr; + checks.check("init rejects null diagnostic slot", Api::init(&handle, nullptr) != 0); + checks.check("invalid init leaves handle null", !handle); + checks.check("init rejects both null slots", Api::init(nullptr, nullptr) != 0); + checks.check("cleanup accepts null", Api::cleanup(nullptr) == 0); + checks.check("depth setter rejects null", Api::depth(nullptr, 1) != 0); + checks.check("argument setter rejects null", Api::arguments(nullptr, 1) != 0); + checks.check("silence setter rejects null", Api::silence(nullptr, 1) != 0); + Parser parser; + TemporaryInput input("null"); + for (unsigned int kind = 0; kind < 3; ++kind) { + ErrorSlot error; + const int rc = kind == 0 ? Api::buffer(nullptr, "{}", 2, &error.value) : + Api::buffer(parser.value, nullptr, kind == 1 ? 0 : 1, &error.value); + checks.check("invalid buffer arguments " + std::to_string(kind), rc != 0); + checks.check("invalid buffer replaces diagnostic " + std::to_string(kind), error.replaced()); + checks.check("invalid buffer diagnostic is null or nonempty " + std::to_string(kind), + !error.value || *error.value); + } + for (const auto& invalid : std::vector>{ + {"null parser", parse_path(nullptr, input.path())}, + {"null filename", parse_path(parser.value, nullptr)}}) { + checks.check("file rejects " + invalid.first, invalid.second.status != 0); + checks.check("file replaces diagnostic for " + invalid.first, invalid.second.replaced); + checks.check("invalid file emits no output: " + invalid.first, invalid.second.output.empty()); + } + checks.check("buffer rejects null diagnostic slot", Api::buffer(parser.value, "{}", 2, nullptr) != 0); + checks.check("file rejects null diagnostic slot", Api::file(parser.value, input.path(), nullptr) != 0); + checks.failure("missing file", parse_path(parser.value, (std::string(input.path()) + ".missing").c_str())); + checks.success("buffer reuse after invalid calls", parse(parser.value, "null"), ": \n"); + checks.success("file reuse after invalid calls", parse(parser.value, "true", true), ": true\n"); +} + +template +void limits(Checks& checks) { + Parser parser; + checks.check("set depth two", Api::depth(parser.value, 2) == 0); + checks.check("set arguments two", Api::arguments(parser.value, 2) == 0); + checks.check("enable silence", Api::silence(parser.value, 1) == 0); + const double upper = static_cast(UINT_MAX); + const std::vector> invalid = { + {"NaN", std::numeric_limits::quiet_NaN()}, + {"infinity", std::numeric_limits::infinity()}, + {"negative infinity", -std::numeric_limits::infinity()}, + {"negative integer", -1.0}, {"negative fraction", -0.5}, + {"fraction below one", 0.5}, {"positive fraction", 1.5}, + {"above UINT_MAX", upper + 1.0} + }; + for (const auto& limit : invalid) { + checks.check("depth rejects " + limit.first, Api::depth(parser.value, limit.second) != 0); + checks.check("argument count rejects " + limit.first, Api::arguments(parser.value, limit.second) != 0); + checks.success("rejected " + limit.first + " preserves limits", parse(parser.value, "{\"a\":[null,true]}")); + checks.failure("rejected " + limit.first + " preserves depth ceiling", parse(parser.value, "{\"a\":[[null]]}")); + checks.failure("rejected " + limit.first + " preserves argument ceiling", parse(parser.value, "[null,true,false]")); + } + checks.check("depth accepts UINT_MAX", Api::depth(parser.value, upper) == 0); + checks.check("argument count accepts UINT_MAX", Api::arguments(parser.value, upper) == 0); + checks.success("maximum limits permit ordinary JSON", parse(parser.value, "{\"a\":[null,true,false]}")); + checks.check("depth accepts zero", Api::depth(parser.value, 0) == 0); + for (const auto& value : {"null", "true", "false", "1", "\"value\""}) + checks.success(std::string("zero depth allows root ") + value, parse(parser.value, value)); + for (const auto& value : {"{}", "[]"}) + checks.failure(std::string("zero depth rejects ") + value, parse(parser.value, value)); + checks.check("restore depth three", Api::depth(parser.value, 3) == 0); + checks.check("argument count accepts zero", Api::arguments(parser.value, 0) == 0); + for (const auto& value : {"{}", "[]", "{\"a\":[{},[]]}"}) + checks.success(std::string("zero arguments allows ") + value, parse(parser.value, value)); + for (const auto& value : {"null", "true", "false", "0", "\"\"", "{\"a\":null}", "[false]"}) + checks.failure(std::string("zero arguments rejects ") + value, parse(parser.value, value)); + checks.check("restore arguments two", Api::arguments(parser.value, 2) == 0); + checks.success("exact depth", parse(parser.value, "[[[null]]]")); + checks.failure("above depth", parse(parser.value, "[[[[null]]]]")); + checks.success("bool and null exactly at count", parse(parser.value, "{\"a\":null,\"b\":false}")); + checks.failure("bool and null above count", parse(parser.value, "[null,true,false]")); + checks.check("negative zero is a valid zero depth", Api::depth(parser.value, -0.0) == 0); + checks.failure("negative zero has zero-limit semantics", parse(parser.value, "{}")); +} + +template +void reuse_and_output(Checks& checks) { + Parser parser; + checks.check("set reuse depth", Api::depth(parser.value, 2) == 0); + checks.check("set reuse arguments", Api::arguments(parser.value, 2) == 0); + for (bool file : {false, true}) { + const std::string kind = file ? "file" : "buffer"; + const Result truncated = parse(parser.value, "{\"leaked\":{\"x\":true,\"y\":", file); + checks.failure(kind + " truncated input", truncated); + checks.check(kind + " emits completed prefix only", truncated.output == "leaked.x: true\n"); + checks.success(kind + " fresh path after failure", parse(parser.value, "{\"fresh\":false}", file), "fresh: false\n"); + checks.success(kind + " root array", parse(parser.value, "[null,true]", file), "array.array: \narray.array: true\n"); + checks.success(kind + " root scalar resets array path", parse(parser.value, "2", file), ": 2\n"); + const Result count = parse(parser.value, "[null,true,false]", file); + checks.failure(kind + " retained argument limit", count); + checks.check(kind + " only permitted arguments printed", count.output == "array.array: \narray.array: true\n"); + checks.success(kind + " resets count after failure", parse(parser.value, "{\"a\":true,\"b\":null}", file), "a: true\nb: \n"); + checks.failure(kind + " retained depth limit", parse(parser.value, "[[[null]]]", file)); + checks.success(kind + " resets after depth failure", parse(parser.value, "{\"after\":null}", file), "after: \n"); + const char binary[] = "a\0b.c\0d: v\0x\na\0b.after: false\n"; + checks.success(kind + " embedded NUL values and paths", parse(parser.value, + "{\"a\\u0000b\":{\"c\\u0000d\":\"v\\u0000x\",\"after\":false}}", file), + std::string(binary, sizeof binary - 1)); + checks.success(kind + " full dotted parent restoration", parse(parser.value, + "{\"a.b\":{\"c.d\":1},\"after\":2}", file), "a.b.c.d: 1\nafter: 2\n"); + checks.success(kind + " EOF root number is finalized", parse(parser.value, "1.250e+3", file), ": 1.250e+3\n"); + checks.failure(kind + " empty input", parse(parser.value, "", file)); + checks.failure(kind + " truncated EOF number", parse(parser.value, "1e+", file)); + } + checks.check("nonzero silence accepted", Api::silence(parser.value, -7) == 0); + checks.success("silent parsing validates and suppresses output", parse(parser.value, "{\"a\":null,\"b\":true}")); + checks.failure("silence retains argument limit", parse(parser.value, "[null,true,false]")); + checks.failure("silence retains syntax validation", parse(parser.value, "{\"a\":true,}")); + checks.success("silence persists across failed file reuse", parse(parser.value, "{\"quiet\":false}", true)); +} + +template +void stream_boundaries(Checks& checks) { + Parser parser; + checks.check("silence boundary failures", Api::silence(parser.value, 1) == 0); + for (const std::size_t offset : {16383u, 16384u, 16385u}) { + const std::string raw_nul = "{}" + std::string(offset - 2, ' ') + std::string(1, '\0') + "{}"; + for (bool file : {false, true}) { + const std::string label = (file ? "file" : "buffer") + + std::string(" raw NUL at offset ") + std::to_string(offset); + const Result result = parse(parser.value, raw_nul, file); + checks.failure(label, result); + checks.check(label + ": no silent output", result.output.empty()); + } + } + const std::string trailing = "{}" + std::string(16382, ' ') + "[]"; + checks.failure("trailing material after file buffer boundary", parse(parser.value, trailing, true)); + checks.failure("trailing material in buffer API", parse(parser.value, trailing)); + checks.check("disable silence for valid boundary output", Api::silence(parser.value, 0) == 0); + const std::string unicode = std::string(16382, ' ') + "\"\xc3\xa4\xf0\x9f\x98\x80\""; + checks.success("UTF-8 spans file buffer boundary", parse(parser.value, unicode, true), ": \xc3\xa4\xf0\x9f\x98\x80\n"); + const std::string escape = std::string(16382, ' ') + "\"\\u0000x\""; + const char binary[] = ": \0x\n"; + checks.success("decoded NUL escape spans file buffer boundary", parse(parser.value, escape, true), + std::string(binary, sizeof binary - 1)); + checks.success("reused adapter after boundary failures", parse(parser.value, "{\"fresh\":true}"), "fresh: true\n"); +} + +template +void run(Checks& checks) { + checks.engine = Api::name(); + const unsigned int initial_total = checks.total(); + const unsigned int initial_failures = checks.failures(); + invalid_arguments(checks); + limits(checks); + reuse_and_output(checks); + stream_boundaries(checks); + const unsigned int total = checks.total() - initial_total; + const unsigned int failures = checks.failures() - initial_failures; + std::printf("%s C API: %u checks passed, %u failed\n", Api::name(), total - failures, failures); +} +} // namespace + +int main() { + try { + Checks checks; +#if HAVE_RAPIDJSON + run(checks); +#endif +#if HAVE_NLOHMANNJSON + run(checks); +#endif + return checks.finish(); + } catch (const std::exception& error) { + std::fprintf(stderr, "FAIL: SAX adapter C API test setup: %s\n", error.what()); + return 1; + } +} +#else +int main() { + std::fprintf(stderr, "SAX adapter C API tests require RapidJSON or NlohmannJSON\n"); + return 77; +} +#endif From e6fe3a35c98b8ceec51edb4bbf01ec7c0b804ec0 Mon Sep 17 00:00:00 2001 From: jens Steinbacher Date: Tue, 13 Oct 2026 14:11:36 +0000 Subject: [PATCH 04/18] Validate remaining native adapters and automate API regression checks --- .gitignore | 12 + Makefile.am | 6 +- configure.ac | 4 +- docs/adapter-stack-benchmark.json | 769 ++++++++++++++++++++++++++++++ docs/adapter-stack.md | 131 +++++ src/Makefile.am | 18 +- src/cjsonparser.c | 170 +++---- src/domarguments.h | 514 ++++++++++++++++++++ src/janssonparser.c | 127 +++-- src/jsoncparser.c | 206 ++++++-- src/jsoncppparser.cpp | 275 +++++++++-- src/simdjsonparser.cpp | 280 ++++++----- src/yyjsonparser.c | 166 +++---- tests/Makefile.am | 46 ++ tests/README.md | 36 +- tests/benchmark_jsoncons.py | 2 +- tests/cpp_adapter_api_tests.cpp | 344 +++++++++++++ tests/dom_adapter_api_tests.c | 174 +++++++ tests/regression_tests.py | 12 +- 19 files changed, 2849 insertions(+), 443 deletions(-) create mode 100644 docs/adapter-stack-benchmark.json create mode 100644 src/domarguments.h create mode 100644 tests/Makefile.am create mode 100644 tests/cpp_adapter_api_tests.cpp create mode 100644 tests/dom_adapter_api_tests.c diff --git a/.gitignore b/.gitignore index 6ef4a37..545443f 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,15 @@ m4/lt~obsolete.m4 # can automatically generate from config.status script # (which is called by configure script)) Makefile + +# Generated API test executables and Automake reports. +/tests/*_api_tests +/tests/pathhandler_tests +/tests/*.log +/tests/*.trs +/tests/__pycache__/ +/build-aux/ +/config.h +/config.h.in~ +/configure~ +/jsonbench-*.tar.gz diff --git a/Makefile.am b/Makefile.am index 515d51b..ff4b8e3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,2 +1,4 @@ -SUBDIRS = src -dist_doc_DATA = README \ No newline at end of file +SUBDIRS = src tests +dist_doc_DATA = README +EXTRA_DIST = docs/adapter-stack.md docs/adapter-stack-benchmark.json \ + docs/jsoncons-sax.md docs/validation-results.md diff --git a/configure.ac b/configure.ac index 1966074..32fd51a 100644 --- a/configure.ac +++ b/configure.ac @@ -3,6 +3,7 @@ AC_PREREQ([2.69]) AC_INIT([jsonbench], [0.2], [airween@gmail.com]) +AC_USE_SYSTEM_EXTENSIONS AC_CONFIG_SRCDIR([config.h.in]) AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_HEADERS([config.h]) @@ -15,6 +16,7 @@ PKGPATH=$($PKG_CONFIG --variable pc_path pkg-config) # Checks for programs. AC_PROG_CXX AC_PROG_CC +AC_CHECK_FUNCS([newlocale uselocale freelocale duplocale]) JSONC_CFLAGS="" JSONC_LDADD="" @@ -696,7 +698,7 @@ AC_CHECK_LIB([m], [powl], [], AC_MSG_ERROR([libm is not installed.], 1)) # Checks for library functions. -AC_CONFIG_FILES([Makefile src/Makefile]) +AC_CONFIG_FILES([Makefile src/Makefile tests/Makefile]) AM_INIT_AUTOMAKE([-Wall -Werror foreign subdir-objects]) AC_MSG_NOTICE([JSON library summary:]) diff --git a/docs/adapter-stack-benchmark.json b/docs/adapter-stack-benchmark.json new file mode 100644 index 0000000..92c6070 --- /dev/null +++ b/docs/adapter-stack-benchmark.json @@ -0,0 +1,769 @@ +{ + "schema_version": 1, + "platform": "Linux-6.18.44-x86_64-with-glibc2.41", + "artifacts": { + "binary": { + "path": "/workspace/digitalwave-jsonbench/src/jsonbench", + "sha256": "06b4c09fc3566a0c102f97a451cfc52382ce5c37d7b1d0e8b7d01970cf5f9c7b" + }, + "config": { + "path": "/workspace/digitalwave-jsonbench/config.h", + "sha256": "0c699c66839ddb24ebd9138ff6ccdfbc7dff21f89038b7b425ac5d1b20cb50e8" + }, + "baseline": { + "path": "/workspace/.jsonbench-tools/jsonbench-dom-baseline", + "sha256": "3e1abf98905ec7f1cf42a97b48b7a2eefa5898c96d143f5bac07dd8af8933bbb" + } + }, + "method": { + "samples": 5, + "warmups": 1, + "cli": "-s -a GENERATED_ARGUMENT_COUNT -n 1 -e ENGINE INPUT", + "input": "root array of identical objects: short ASCII string, exact integer, boolean", + "timing": "CLI parser elapsed seconds; independent fresh processes after warmups", + "rss": "Linux wait4 child peak RSS in KiB via a small compiled C launcher", + "file_buffer_size": 524288, + "yajl_input": "attempt every requested size; streaming adapters have no FILE_BUFFER_SIZE file cap", + "compiler": [ + "cc" + ], + "caveat": "Original DOM adapter only parses/discards a DOM. SAX also builds argument paths and enforces argument/size limits. Synthetic cached-file workload; no ModSecurity performance conclusion." + }, + "inputs": [ + { + "target_bytes": 409600, + "bytes": 409564, + "objects": 5319, + "arguments": 15957, + "sha256": "44674cc69b5e076c937e298d937305cd9e94247acb815790977ab4c54a7c06cb" + }, + { + "target_bytes": 5242880, + "bytes": 5242854, + "objects": 68089, + "arguments": 204267, + "sha256": "07d23f5e5b8f3d65dcc9143ce41c6e7ca16b1fde5af69d8aa279247d92d8c9bc" + }, + { + "target_bytes": 33554432, + "bytes": 33554368, + "objects": 435771, + "arguments": 1307313, + "sha256": "47932b32f3c7a4dcc2f1b736cad541044911399454d5565f090b67f1978d97d7" + } + ], + "runs": [ + { + "variant": "jsoncons_dom", + "input_bytes": 409564, + "status": "ok", + "warmups": [ + { + "status": "ok", + "parser_seconds": 0.00834565, + "wall_seconds": 0.012779053000485874, + "peak_rss_kib": 5232, + "user_seconds": 0.006377, + "system_seconds": 0.004688, + "stderr": "JSONCONS does not support max depth limit\nJSONCONS does not support max arg num limit" + } + ], + "samples": [ + { + "status": "ok", + "parser_seconds": 0.005942113, + "wall_seconds": 0.010221734000879223, + "peak_rss_kib": 5332, + "user_seconds": 0.004348, + "system_seconds": 0.003788, + "stderr": "JSONCONS does not support max depth limit\nJSONCONS does not support max arg num limit" + }, + { + "status": "ok", + "parser_seconds": 0.004869709, + "wall_seconds": 0.008874135999576538, + "peak_rss_kib": 5352, + "user_seconds": 0.003706, + "system_seconds": 0.003955, + "stderr": "JSONCONS does not support max depth limit\nJSONCONS does not support max arg num limit" + }, + { + "status": "ok", + "parser_seconds": 0.00533557, + "wall_seconds": 0.008604189999459777, + "peak_rss_kib": 5360, + "user_seconds": 0.007672, + "system_seconds": 0.0, + "stderr": "JSONCONS does not support max depth limit\nJSONCONS does not support max arg num limit" + }, + { + "status": "ok", + "parser_seconds": 0.004097096, + "wall_seconds": 0.007158143000197015, + "peak_rss_kib": 5300, + "user_seconds": 0.0, + "system_seconds": 0.006136, + "stderr": "JSONCONS does not support max depth limit\nJSONCONS does not support max arg num limit" + }, + { + "status": "ok", + "parser_seconds": 0.00524869, + "wall_seconds": 0.010258599000735558, + "peak_rss_kib": 5376, + "user_seconds": 0.0, + "system_seconds": 0.009088, + "stderr": "JSONCONS does not support max depth limit\nJSONCONS does not support max arg num limit" + } + ], + "summary": { + "parser_seconds": { + "mean": 0.0050986356, + "sample_stdev": 0.0006792638139429335, + "min": 0.004097096, + "max": 0.005942113 + }, + "wall_seconds": { + "mean": 0.009023360400169622, + "sample_stdev": 0.0012883052222716844, + "min": 0.007158143000197015, + "max": 0.010258599000735558 + }, + "peak_rss_kib": { + "mean": 5344, + "sample_stdev": 29.257477676655586, + "min": 5300, + "max": 5376 + } + } + }, + { + "variant": "jsoncons_sax", + "input_bytes": 409564, + "status": "ok", + "warmups": [ + { + "status": "ok", + "parser_seconds": 0.003161407, + "wall_seconds": 0.0067568189988378435, + "peak_rss_kib": 2764, + "user_seconds": 0.002657, + "system_seconds": 0.002666 + } + ], + "samples": [ + { + "status": "ok", + "parser_seconds": 0.002873044, + "wall_seconds": 0.005499205999512924, + "peak_rss_kib": 2740, + "user_seconds": 0.004632, + "system_seconds": 0.0 + }, + { + "status": "ok", + "parser_seconds": 0.00277684, + "wall_seconds": 0.005453899000713136, + "peak_rss_kib": 2700, + "user_seconds": 0.004666, + "system_seconds": 0.0 + }, + { + "status": "ok", + "parser_seconds": 0.00461121, + "wall_seconds": 0.00773739499891235, + "peak_rss_kib": 2760, + "user_seconds": 0.002508, + "system_seconds": 0.004315 + }, + { + "status": "ok", + "parser_seconds": 0.003301368, + "wall_seconds": 0.005965616999674239, + "peak_rss_kib": 2816, + "user_seconds": 0.005132, + "system_seconds": 0.0 + }, + { + "status": "ok", + "parser_seconds": 0.002905733, + "wall_seconds": 0.006117556999015505, + "peak_rss_kib": 2812, + "user_seconds": 0.005262, + "system_seconds": 0.0 + } + ], + "summary": { + "parser_seconds": { + "mean": 0.003293639, + "sample_stdev": 0.0007632996630983143, + "min": 0.00277684, + "max": 0.00461121 + }, + "wall_seconds": { + "mean": 0.006154734799565631, + "sample_stdev": 0.0009304367069201241, + "min": 0.005453899000713136, + "max": 0.00773739499891235 + }, + "peak_rss_kib": { + "mean": 2765.6, + "sample_stdev": 49.20162598939186, + "min": 2700, + "max": 2816 + } + } + }, + { + "variant": "yajl", + "input_bytes": 409564, + "status": "ok", + "warmups": [ + { + "status": "ok", + "parser_seconds": 0.002089875, + "wall_seconds": 0.005340036999768927, + "peak_rss_kib": 2640, + "user_seconds": 0.004199, + "system_seconds": 0.0 + } + ], + "samples": [ + { + "status": "ok", + "parser_seconds": 0.00253173, + "wall_seconds": 0.005423010999948019, + "peak_rss_kib": 2624, + "user_seconds": 0.004345, + "system_seconds": 0.0 + }, + { + "status": "ok", + "parser_seconds": 0.001685196, + "wall_seconds": 0.004349725999418297, + "peak_rss_kib": 2596, + "user_seconds": 0.003362, + "system_seconds": 0.0 + }, + { + "status": "ok", + "parser_seconds": 0.001964957, + "wall_seconds": 0.004700282999692718, + "peak_rss_kib": 2656, + "user_seconds": 0.0, + "system_seconds": 0.003939 + }, + { + "status": "ok", + "parser_seconds": 0.002041793, + "wall_seconds": 0.004723529000330018, + "peak_rss_kib": 2680, + "user_seconds": 0.003755, + "system_seconds": 0.0 + }, + { + "status": "ok", + "parser_seconds": 0.002082905, + "wall_seconds": 0.0048981210002239095, + "peak_rss_kib": 2656, + "user_seconds": 0.00415, + "system_seconds": 0.0 + } + ], + "summary": { + "parser_seconds": { + "mean": 0.0020613161999999997, + "sample_stdev": 0.00030532272889796457, + "min": 0.001685196, + "max": 0.00253173 + }, + "wall_seconds": { + "mean": 0.004818933999922592, + "sample_stdev": 0.0003919553305277555, + "min": 0.004349725999418297, + "max": 0.005423010999948019 + }, + "peak_rss_kib": { + "mean": 2642.4, + "sample_stdev": 32.69250678672408, + "min": 2596, + "max": 2680 + } + } + }, + { + "variant": "jsoncons_dom", + "input_bytes": 5242854, + "status": "ok", + "warmups": [ + { + "status": "ok", + "parser_seconds": 0.068317148, + "wall_seconds": 0.07272912799999176, + "peak_rss_kib": 39016, + "user_seconds": 0.062874, + "system_seconds": 0.008857, + "stderr": "JSONCONS does not support max depth limit\nJSONCONS does not support max arg num limit" + } + ], + "samples": [ + { + "status": "ok", + "parser_seconds": 0.067531686, + "wall_seconds": 0.07280279899896414, + "peak_rss_kib": 39048, + "user_seconds": 0.056217, + "system_seconds": 0.0113, + "stderr": "JSONCONS does not support max depth limit\nJSONCONS does not support max arg num limit" + }, + { + "status": "ok", + "parser_seconds": 0.066968959, + "wall_seconds": 0.07150823099982517, + "peak_rss_kib": 39024, + "user_seconds": 0.057439, + "system_seconds": 0.012905, + "stderr": "JSONCONS does not support max depth limit\nJSONCONS does not support max arg num limit" + }, + { + "status": "ok", + "parser_seconds": 0.067843485, + "wall_seconds": 0.0728891399994609, + "peak_rss_kib": 39004, + "user_seconds": 0.054984, + "system_seconds": 0.01637, + "stderr": "JSONCONS does not support max depth limit\nJSONCONS does not support max arg num limit" + }, + { + "status": "ok", + "parser_seconds": 0.063197693, + "wall_seconds": 0.06810393699925044, + "peak_rss_kib": 39020, + "user_seconds": 0.048484, + "system_seconds": 0.018276, + "stderr": "JSONCONS does not support max depth limit\nJSONCONS does not support max arg num limit" + }, + { + "status": "ok", + "parser_seconds": 0.068296128, + "wall_seconds": 0.07449504699980025, + "peak_rss_kib": 39048, + "user_seconds": 0.056787, + "system_seconds": 0.015881, + "stderr": "JSONCONS does not support max depth limit\nJSONCONS does not support max arg num limit" + } + ], + "summary": { + "parser_seconds": { + "mean": 0.06676759019999999, + "sample_stdev": 0.0020532008687933322, + "min": 0.063197693, + "max": 0.068296128 + }, + "wall_seconds": { + "mean": 0.07195983079946018, + "sample_stdev": 0.0024017385127535207, + "min": 0.06810393699925044, + "max": 0.07449504699980025 + }, + "peak_rss_kib": { + "mean": 39028.8, + "sample_stdev": 19.05780679931455, + "min": 39004, + "max": 39048 + } + } + }, + { + "variant": "jsoncons_sax", + "input_bytes": 5242854, + "status": "ok", + "warmups": [ + { + "status": "ok", + "parser_seconds": 0.037217954, + "wall_seconds": 0.04177322000032291, + "peak_rss_kib": 2700, + "user_seconds": 0.035879, + "system_seconds": 0.00433 + } + ], + "samples": [ + { + "status": "ok", + "parser_seconds": 0.041133948, + "wall_seconds": 0.04582990400012932, + "peak_rss_kib": 2756, + "user_seconds": 0.034996, + "system_seconds": 0.009235 + }, + { + "status": "ok", + "parser_seconds": 0.044249468, + "wall_seconds": 0.0479045780011802, + "peak_rss_kib": 2808, + "user_seconds": 0.046605, + "system_seconds": 0.0 + }, + { + "status": "ok", + "parser_seconds": 0.042284869, + "wall_seconds": 0.04667604799942637, + "peak_rss_kib": 2676, + "user_seconds": 0.040573, + "system_seconds": 0.004851 + }, + { + "status": "ok", + "parser_seconds": 0.041194618, + "wall_seconds": 0.04554033000022173, + "peak_rss_kib": 2700, + "user_seconds": 0.042826, + "system_seconds": 0.001293 + }, + { + "status": "ok", + "parser_seconds": 0.041519457, + "wall_seconds": 0.04507738199936284, + "peak_rss_kib": 2760, + "user_seconds": 0.039701, + "system_seconds": 0.004262 + } + ], + "summary": { + "parser_seconds": { + "mean": 0.042076472000000004, + "sample_stdev": 0.001298231932643586, + "min": 0.041133948, + "max": 0.044249468 + }, + "wall_seconds": { + "mean": 0.046205648400064095, + "sample_stdev": 0.0011140528567836474, + "min": 0.04507738199936284, + "max": 0.0479045780011802 + }, + "peak_rss_kib": { + "mean": 2740, + "sample_stdev": 52.38320341483518, + "min": 2676, + "max": 2808 + } + } + }, + { + "variant": "yajl", + "input_bytes": 5242854, + "status": "ok", + "warmups": [ + { + "status": "ok", + "parser_seconds": 0.025824325, + "wall_seconds": 0.03000063100080297, + "peak_rss_kib": 2668, + "user_seconds": 0.024324, + "system_seconds": 0.004382 + } + ], + "samples": [ + { + "status": "ok", + "parser_seconds": 0.024768978, + "wall_seconds": 0.030732313000044087, + "peak_rss_kib": 2660, + "user_seconds": 0.027236, + "system_seconds": 0.002323 + }, + { + "status": "ok", + "parser_seconds": 0.025020697, + "wall_seconds": 0.02879873299934843, + "peak_rss_kib": 2576, + "user_seconds": 0.024873, + "system_seconds": 0.002519 + }, + { + "status": "ok", + "parser_seconds": 0.023725027, + "wall_seconds": 0.027300577999994857, + "peak_rss_kib": 2540, + "user_seconds": 0.025996, + "system_seconds": 0.0 + }, + { + "status": "ok", + "parser_seconds": 0.023904197, + "wall_seconds": 0.02741518099901441, + "peak_rss_kib": 2512, + "user_seconds": 0.02223, + "system_seconds": 0.003988 + }, + { + "status": "ok", + "parser_seconds": 0.025889924, + "wall_seconds": 0.030059959999562125, + "peak_rss_kib": 2672, + "user_seconds": 0.02339, + "system_seconds": 0.004708 + } + ], + "summary": { + "parser_seconds": { + "mean": 0.0246617646, + "sample_stdev": 0.0008803534364874724, + "min": 0.023725027, + "max": 0.025889924 + }, + "wall_seconds": { + "mean": 0.028861352999592783, + "sample_stdev": 0.0015385466667216833, + "min": 0.027300577999994857, + "max": 0.030732313000044087 + }, + "peak_rss_kib": { + "mean": 2592, + "sample_stdev": 71.386273190299, + "min": 2512, + "max": 2672 + } + } + }, + { + "variant": "jsoncons_dom", + "input_bytes": 33554368, + "status": "ok", + "warmups": [ + { + "status": "ok", + "parser_seconds": 0.394697364, + "wall_seconds": 0.4022609969997575, + "peak_rss_kib": 228140, + "user_seconds": 0.321515, + "system_seconds": 0.079382, + "stderr": "JSONCONS does not support max depth limit\nJSONCONS does not support max arg num limit" + } + ], + "samples": [ + { + "status": "ok", + "parser_seconds": 0.398498366, + "wall_seconds": 0.40967574900059844, + "peak_rss_kib": 228160, + "user_seconds": 0.309652, + "system_seconds": 0.098612, + "stderr": "JSONCONS does not support max depth limit\nJSONCONS does not support max arg num limit" + }, + { + "status": "ok", + "parser_seconds": 0.502713831, + "wall_seconds": 0.5109430359989346, + "peak_rss_kib": 228184, + "user_seconds": 0.40598, + "system_seconds": 0.101394, + "stderr": "JSONCONS does not support max depth limit\nJSONCONS does not support max arg num limit" + }, + { + "status": "ok", + "parser_seconds": 0.50657986, + "wall_seconds": 0.5165568050015281, + "peak_rss_kib": 228228, + "user_seconds": 0.440298, + "system_seconds": 0.074468, + "stderr": "JSONCONS does not support max depth limit\nJSONCONS does not support max arg num limit" + }, + { + "status": "ok", + "parser_seconds": 0.488561492, + "wall_seconds": 0.5001273090001632, + "peak_rss_kib": 228216, + "user_seconds": 0.406979, + "system_seconds": 0.091559, + "stderr": "JSONCONS does not support max depth limit\nJSONCONS does not support max arg num limit" + }, + { + "status": "ok", + "parser_seconds": 0.49204873, + "wall_seconds": 0.4997036709992244, + "peak_rss_kib": 228184, + "user_seconds": 0.400727, + "system_seconds": 0.097148, + "stderr": "JSONCONS does not support max depth limit\nJSONCONS does not support max arg num limit" + } + ], + "summary": { + "parser_seconds": { + "mean": 0.4776804558, + "sample_stdev": 0.04487898459854655, + "min": 0.398498366, + "max": 0.50657986 + }, + "wall_seconds": { + "mean": 0.4874013140000898, + "sample_stdev": 0.04404207568102363, + "min": 0.40967574900059844, + "max": 0.5165568050015281 + }, + "peak_rss_kib": { + "mean": 228194.4, + "sample_stdev": 27.36421020237931, + "min": 228160, + "max": 228228 + } + } + }, + { + "variant": "jsoncons_sax", + "input_bytes": 33554368, + "status": "ok", + "warmups": [ + { + "status": "ok", + "parser_seconds": 0.312678616, + "wall_seconds": 0.3172133509997366, + "peak_rss_kib": 2680, + "user_seconds": 0.30775, + "system_seconds": 0.00789 + } + ], + "samples": [ + { + "status": "ok", + "parser_seconds": 0.268272694, + "wall_seconds": 0.2736984000002849, + "peak_rss_kib": 2688, + "user_seconds": 0.265453, + "system_seconds": 0.006499 + }, + { + "status": "ok", + "parser_seconds": 0.289150014, + "wall_seconds": 0.2934957050001685, + "peak_rss_kib": 2680, + "user_seconds": 0.2805, + "system_seconds": 0.011343 + }, + { + "status": "ok", + "parser_seconds": 0.282361849, + "wall_seconds": 0.2875684549999278, + "peak_rss_kib": 2628, + "user_seconds": 0.27292, + "system_seconds": 0.011781 + }, + { + "status": "ok", + "parser_seconds": 0.312631015, + "wall_seconds": 0.31624293999993824, + "peak_rss_kib": 2768, + "user_seconds": 0.315019, + "system_seconds": 0.0 + }, + { + "status": "ok", + "parser_seconds": 0.268829483, + "wall_seconds": 0.27311448300133634, + "peak_rss_kib": 2776, + "user_seconds": 0.267391, + "system_seconds": 0.004076 + } + ], + "summary": { + "parser_seconds": { + "mean": 0.284249011, + "sample_stdev": 0.018207992136034277, + "min": 0.268272694, + "max": 0.312631015 + }, + "wall_seconds": { + "mean": 0.28882399660033115, + "sample_stdev": 0.017683128903411338, + "min": 0.27311448300133634, + "max": 0.31624293999993824 + }, + "peak_rss_kib": { + "mean": 2708, + "sample_stdev": 62.864934582006846, + "min": 2628, + "max": 2776 + } + } + }, + { + "variant": "yajl", + "input_bytes": 33554368, + "status": "ok", + "warmups": [ + { + "status": "ok", + "parser_seconds": 0.162595989, + "wall_seconds": 0.16703827600031218, + "peak_rss_kib": 2512, + "user_seconds": 0.162094, + "system_seconds": 0.003421 + } + ], + "samples": [ + { + "status": "ok", + "parser_seconds": 0.177104936, + "wall_seconds": 0.18118300400055887, + "peak_rss_kib": 2560, + "user_seconds": 0.171672, + "system_seconds": 0.008364 + }, + { + "status": "ok", + "parser_seconds": 0.161486119, + "wall_seconds": 0.16535158700025931, + "peak_rss_kib": 2664, + "user_seconds": 0.151782, + "system_seconds": 0.012155 + }, + { + "status": "ok", + "parser_seconds": 0.196355569, + "wall_seconds": 0.20075590200031002, + "peak_rss_kib": 2648, + "user_seconds": 0.184412, + "system_seconds": 0.014947 + }, + { + "status": "ok", + "parser_seconds": 0.221555616, + "wall_seconds": 0.2261696210007358, + "peak_rss_kib": 2652, + "user_seconds": 0.224313, + "system_seconds": 0.0 + }, + { + "status": "ok", + "parser_seconds": 0.18815985, + "wall_seconds": 0.19362888499927067, + "peak_rss_kib": 2668, + "user_seconds": 0.190911, + "system_seconds": 0.000588 + } + ], + "summary": { + "parser_seconds": { + "mean": 0.188932418, + "sample_stdev": 0.022434135480826727, + "min": 0.161486119, + "max": 0.221555616 + }, + "wall_seconds": { + "mean": 0.19341779980022694, + "sample_stdev": 0.022715504737969516, + "min": 0.16535158700025931, + "max": 0.2261696210007358 + }, + "peak_rss_kib": { + "mean": 2638.4, + "sample_stdev": 44.595963942939946, + "min": 2560, + "max": 2668 + } + } + } + ], + "status": "ok" +} diff --git a/docs/adapter-stack.md b/docs/adapter-stack.md index 2bd41d3..bbb7443 100644 --- a/docs/adapter-stack.md +++ b/docs/adapter-stack.md @@ -94,6 +94,137 @@ g++ -std=c++17 -Wall -Wextra -Werror $CPPFLAGS -I. \ /tmp/sax-api-tests ``` +## 3. Native DOM and simdjson validation adapters + +JSON-C, Jansson, cJSON, JsonCpp and yyjson retain their native DOM APIs. They now +read files dynamically, require complete native grammar validation, check +Unicode/NUL/resource boundaries, enforce depth and scalar counts, retain source +order where possible, and report unsupported native representations explicitly. +They still allocate document-sized input and DOM memory. A narrow shared C +preflight bounds hostile tokens/depth and checks native numeric ranges; it is +not a substitute for native JSON grammar parsing. An iterative shared walker +uses length-aware structural paths. Borrowed raw numeric spans are matched to +native numeric leaves before successful output, avoiding decimal reformatting +without bypassing native range checks. A mixed numeric/string golden fixture +checks that numeric-looking keys and strings do not consume those spans. + +JSON-C's strict 0.18 reader rejects raw multibyte characters; after validating +Unicode scalars, this adapter transcodes them to equivalent Unicode escapes +while keeping native strict/UTF-8 flags enabled. JSON-C and Jansson share +incompatible public symbol names: the executable links Jansson first and uses +JSON-C linkhash traversal to avoid ambiguous iterator calls. Their C APIs +cannot retain all required duplicate/NUL-key information, so these cases are +rejected instead of silently producing incorrect arguments. cJSON likewise +rejects decoded NUL and integers outside its lossless double range. JsonCpp +rejects duplicates, uses source offsets for order/raw numbers, and has a native +1023-container ceiling. + +simdjson uses public OnDemand traversal, with native numeric validation and raw +numeric output; it does not build a DOM. It does retain padded whole-document +input and a structural index, so it is not a bounded-input SAX stream. Its +recursive adapter has an independent 512-container safety ceiling, including +when the caller increases the configured depth. The default limit of 500 is +unchanged. Committed API tests exercise 500/512 and rejection at 513/999/2000 +under an 8 MiB stack and sanitizers; JsonCpp at 999 is also exercised. + +JSON-C, Jansson and cJSON isolate native decimal parsing in a temporary POSIX +thread numeric locale and restore the caller locale on success and every error +path. Other locale categories remain intact. If POSIX locale functions are +unavailable, an existing dot-decimal locale still works; a comma-decimal locale +is explicitly unsupported rather than changed globally. Tests cover process +and thread decimal-comma locales, malformed native grammar, preflight failures, +missing files, silence and exact decimal output. + +The iteration fixture now uses distinct keys, isolating reset/output behavior. +Duplicate keys remain a separate required golden test for every engine. + +## Integrated verification + +The standard `./autogen.sh`, `./configure`, `make` build passed with ten engines. +Installed native versions: JSON-C 0.18, Jansson 2.14, cJSON 1.7.18, JsonCpp 1.9.6, +simdjson 3.12.3 and yyjson 0.10.0 (alongside YAJL 2.1, RapidJSON 1.1, +nlohmann 3.11 and jsoncons 1.3.2). +Glaze is unavailable in this environment and was not tested; optional dependency +handling remains intact. The complete CLI run returned **1**, correctly keeping +all native representation limitations visible: + +| Engine | Passed | Failed | Remaining required failures | +| --- | ---: | ---: | --- | +| YAJL | 235 | 0 | None | +| RapidJSON | 231 | 4 | Extreme positive floating exponents | +| nlohmann | 231 | 4 | Extreme positive floating exponents | +| JSON-C | 225 | 10 | Duplicate keys, NUL keys, native numeric ranges | +| Jansson | 225 | 10 | Duplicate keys, NUL keys, native numeric ranges | +| cJSON | 227 | 8 | Decoded NUL, native numeric ranges | +| JsonCpp | 233 | 2 | Duplicate keys | +| jsoncons | 235 | 0 | None | +| simdjson | 229 | 6 | Native integer/floating ranges | +| yyjson | 235 | 0 | None | + +Each failing valid case is tested in output and silent mode. Unsupported ranges +produce diagnostics, with no silent integer rounding. Assertions are unchanged +for the required contract. There were **2339 passes, 44 failures, 0 skips**, plus +**2 separately checked numeric spelling differences**. This includes 33 exact +jsoncons/YAJL comparisons. `tests/run_tests.sh JSONCONS` returned 0 with **268 +passes, 0 failures, 0 skips, 2 differences**; the YAJL run returned 0 with **235 +passes**. No crash or timeout was reported. + +`make check` runs the six direct API/path harnesses. They contain 181 PathHandler, +220 jsoncons, 3900 YAJL, 582 SAX-pair, 565 C DOM and 331 C++-pair checks. These +harnesses also passed ASan/UBSan with leak detection. An all-engines-disabled +configuration builds successfully and reports PathHandler PASS and five API +SKIPs, with no failures. Source distribution checks include the harnesses, +fixtures and helpers. The CLI runner remains independent and supports selecting +one engine or another executable. + +The integrated ASan/UBSan build also passed all six API harnesses and reproduced +the exact CLI counts above. A separate sanitizer log pass checks expected +rejections for hidden reports, as a native rejection and a sanitizer can both +exit nonzero. The sanitizer run used O1, frame pointers, non-PIE executables, +leak detection, and all ten available engines. + +## Measured jsoncons/YAJL comparison + +The benchmark helper now attempts every requested YAJL input size. With five +fresh-process samples after one warmup per variant, the current binary and the +preserved original DOM executable produced: + +| Input | Original jsoncons DOM (ms / KiB RSS) | jsoncons SAX (ms / KiB RSS) | YAJL streaming (ms / KiB RSS) | +| --- | --- | --- | --- | +| 409564 bytes | 5.099 ± 0.679 / 5344 | 3.294 ± 0.763 / 2766 | 2.061 ± 0.305 / 2642 | +| 5242854 bytes | 66.768 ± 2.053 / 39029 | 42.076 ± 1.298 / 2740 | 24.662 ± 0.880 / 2592 | +| 33554368 bytes | 477.680 ± 44.879 / 228194 | 284.249 ± 18.208 / 2708 | 188.932 ± 22.434 / 2638 | + +Times are mean parser time ± sample standard deviation; RSS is mean peak child +RSS. Linux/GCC 14.2 with effective O2; short ASCII strings, integers and booleans +in a root array, silent mode and exact generated argument limit. Binary/config +hashes were checked for changes during measurement. The original DOM adapter +only parses/discards a DOM; SAX also builds paths and enforces limits. These +cached synthetic workloads do not establish ModSecurity throughput. The raw +report is committed as [adapter-stack-benchmark.json](adapter-stack-benchmark.json), +including raw samples, input/binary/config hashes and environment metadata. +Reproduce with: + +```sh +python3 tests/benchmark_jsoncons.py --baseline /path/to/original/jsonbench \ + --samples 5 --sizes 400KiB 5MiB 32MiB --output /tmp/benchmark.json +``` + +For ModSecurity, prioritize the jsoncons SAX adapter for bounded-input, +lossless-number evaluation, then test request-body chunk boundaries, transaction +argument insertion, partial-output rollback after errors, and request-wide +resource limits in an independently licensed integration. YAJL now provides a +safer compatibility reference. RapidJSON/nlohmann remain SAX alternatives with +native exponent limits; simdjson is an OnDemand alternative with whole-input +memory. None of these changes implements JSON Schema evaluation. + +Partial SAX arguments can be emitted before a later parse error; an embedding +must discard them for a failed document. DOM adapters require whole-document +memory, so request-wide input limits remain necessary. The multi-library link +order and POSIX locale fallback were checked on Linux; other platforms and +library upgrades need their own build/ABI validation. No separate ModSecurity +integration or fuzzing campaign has been performed here. + ## Optional JSON Schema validation No schema validator dependency is introduced. Schema rules are a separate diff --git a/src/Makefile.am b/src/Makefile.am index 371c6d3..75d7f0b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -28,6 +28,22 @@ jsonbench_CPPFLAGS = $(YAJL_CFLAGS) $(JSONC_CFLAGS) $(JANSSON_CFLAGS) \ jsonbench_CFLAGS = $(AM_CFLAGS) jsonbench_CXXFLAGS = $(AM_CXXFLAGS) $(GLAZE_CXXFLAGS) -jsonbench_LDADD = $(YAJL_LDADD) $(JSONC_LDADD) $(JANSSON_LDADD) \ +jsonbench_LDADD = $(YAJL_LDADD) $(JANSSON_LDADD) $(JSONC_LDADD) \ $(CJSON_LDADD) $(JSONCPP_LDADD) $(SIMDJSON_LDADD) \ $(YYJSON_LDADD) -lm + +# Jansson must precede JSON-C for overlapping unversioned object APIs. +# JSON-C traversal uses linkhash to avoid ambiguous iterator calls. +noinst_HEADERS = cjsonparser.h \ + domarguments.h \ + glazeparser.h \ + janssonparser.h \ + jsonconsparser.h \ + jsoncparser.h \ + jsoncppparser.h \ + nlparser.h \ + pathhandler.h \ + rjparser.h \ + simdjsonparser.h \ + yajlparser.h \ + yyjsonparser.h diff --git a/src/cjsonparser.c b/src/cjsonparser.c index 4571174..754b08f 100644 --- a/src/cjsonparser.c +++ b/src/cjsonparser.c @@ -1,123 +1,103 @@ #include "cjsonparser.h" - #if HAVE_CJSON - -#include -#include -#include -#include +#include "domarguments.h" #include -struct cjson_parser { - int silence; -}; - -static int read_whole_file(const char *filename, char **content, long *length, char **error_msg) { - FILE *fp = fopen(filename, "rb"); - if (fp == NULL) { - *error_msg = strdup("Unable to open JSON file"); - return 1; - } - - if (fseek(fp, 0, SEEK_END) != 0) { - fclose(fp); - *error_msg = strdup("Unable to seek JSON file"); - return 1; - } - - long len = ftell(fp); - if (len < 0) { - fclose(fp); - *error_msg = strdup("Unable to measure JSON file"); - return 1; - } - rewind(fp); - - char *buffer = (char *)malloc((size_t)len + 1U); - if (buffer == NULL) { - fclose(fp); - *error_msg = strdup("Unable to allocate JSON buffer"); - return 1; - } +struct cjson_parser { dom_settings settings; }; - size_t read_len = fread(buffer, 1, (size_t)len, fp); - fclose(fp); - if (read_len != (size_t)len) { - free(buffer); - *error_msg = strdup("Unable to read entire JSON file"); - return 1; - } - - buffer[len] = '\0'; - *content = buffer; - *length = len; - return 0; +static int cjson_kind(const void *node) { + return cJSON_IsObject((const cJSON *)node) ? DOM_OBJECT : cJSON_IsArray((const cJSON *)node) ? DOM_ARRAY : DOM_SCALAR; } -int cjson_json_init(cjson_parser **parser, char **error_msg) { - assert(parser != NULL); - assert(error_msg != NULL); - - *parser = (cjson_parser *)calloc(1, sizeof(cjson_parser)); - if (*parser == NULL) { - *error_msg = strdup("Unable to allocate cJSON parser"); - return 1; +static int cjson_child(const void *node, dom_cursor *cursor, int first, + const void **child, const char **key, size_t *key_length) { + const cJSON *json = (const cJSON *)node; + const cJSON *item = first ? json->child : ((const cJSON *)cursor->pointer)->next; + if (item == NULL) return 0; + cursor->pointer = (void *)item; *child = item; + if (cJSON_IsObject(json)) { + *key = item->string; *key_length = strlen(item->string); } + return 1; +} - *error_msg = NULL; +static int cjson_scalar(const void *node, dom_value *value, char **error) { + const cJSON *json = (const cJSON *)node; + if (cJSON_IsNull(json)) { value->data = ""; value->length = 0; } + else if (cJSON_IsBool(json)) { + value->data = cJSON_IsTrue(json) ? "true" : "false"; value->length = strlen(value->data); + } else if (cJSON_IsString(json)) { + value->data = json->valuestring; value->length = strlen(value->data); + } else if (cJSON_IsNumber(json)) { + int length; + if (!isfinite(json->valuedouble)) return dom_error(error, "Nonfinite cJSON number"); + length = snprintf(value->number, sizeof(value->number), "%.17g", json->valuedouble); + if (length < 0 || (size_t)length >= sizeof(value->number)) return dom_error(error, "Unable to format cJSON number"); + value->data = value->number; value->length = (size_t)length; + value->number_kind = DOM_ANY_NUMBER; + } else return dom_error(error, "Unexpected cJSON scalar type"); return 0; } -int cjson_parse_file(cjson_parser *parser, const char *filename, char **error_msg) { - (void)parser; - if (filename == NULL || error_msg == NULL) { - return -1; - } +static const dom_adapter cjson_adapter = {cjson_kind, cjson_child, cjson_scalar}; +int cjson_parse_file(cjson_parser *parser, const char *filename, char **error_msg) { char *content = NULL; - long length = 0; - int rc = read_whole_file(filename, &content, &length, error_msg); - if (rc != 0) { - return 2; - } - - cJSON *root = cJSON_Parse(content); - free(content); - + size_t length = 0, keys = 0; + dom_numbers numbers = {0}; + dom_locale_guard locale = {0}; + const char *end = NULL; + cJSON *root = NULL; + int rc; + if (error_msg == NULL) return -1; + *error_msg = NULL; + if (parser == NULL || filename == NULL) return dom_error(error_msg, "Invalid cJSON parser arguments"); + rc = dom_locale_begin(&locale, error_msg); + if (rc != 0) return rc; + rc = dom_read_file(filename, &content, &length, error_msg); + if (rc != 0) goto done; + /* cJSON has no decoded string lengths and stores all numbers as doubles. + * Reject NUL and unsafe integers instead of silently truncating/rounding. */ + rc = dom_preflight_numbers(content, length, &parser->settings, DOM_NUL_FORBIDDEN, + DOM_NUM_DOUBLE, &keys, &numbers, error_msg); + if (rc != 0) goto done; + root = cJSON_ParseWithLengthOpts(content, length + 1, &end, 1); if (root == NULL) { - const char *cjson_error = cJSON_GetErrorPtr(); - if (cjson_error != NULL) { - *error_msg = strdup(cjson_error); - } else { - *error_msg = strdup("cJSON_Parse failed"); - } - return 2; + char message[128]; + /* Error pointers refer into content: capture position before freeing. */ + size_t offset = end != NULL && end >= content && end <= content + length ? (size_t)(end - content) : 0; + snprintf(message, sizeof(message), "cJSON parse error at byte %zu", offset); + rc = dom_error(error_msg, message); goto done; } - + rc = dom_walk_numbers(root, &cjson_adapter, &parser->settings, &keys, &numbers, error_msg); + done: cJSON_Delete(root); + free(numbers.spans); + free(content); + return dom_locale_finish(&locale, rc, error_msg); +} + +int cjson_json_init(cjson_parser **parser, char **error_msg) { + if (error_msg == NULL) return -1; + *error_msg = NULL; + if (parser == NULL) return dom_error(error_msg, "Invalid cJSON parser output pointer"); + *parser = (cjson_parser *)calloc(1, sizeof(**parser)); + if (*parser == NULL) return dom_error(error_msg, "Unable to allocate cJSON parser"); + dom_settings_init(&(*parser)->settings); return 0; } -int cjson_set_max_depth(cjson_parser *parser, double max_depth) { - if (parser == NULL) { - return -1; - } - (void)max_depth; - return 1; +int cjson_set_max_depth(cjson_parser *parser, double value) { + return parser == NULL ? -1 : dom_set_limit(value, &parser->settings.max_depth); } -int cjson_set_max_arg_num(cjson_parser *parser, double max_arg_num) { - if (parser == NULL) { - return -1; - } - (void)max_arg_num; - return 1; +int cjson_set_max_arg_num(cjson_parser *parser, double value) { + return parser == NULL ? -1 : dom_set_limit(value, &parser->settings.max_args); } int cjson_set_silence(cjson_parser *parser, int silence) { - if (parser != NULL) { - parser->silence = silence; - } + if (parser == NULL) return -1; + parser->settings.silence = silence != 0; return 0; } diff --git a/src/domarguments.h b/src/domarguments.h new file mode 100644 index 0000000..aed6cd1 --- /dev/null +++ b/src/domarguments.h @@ -0,0 +1,514 @@ +#ifndef JSONBENCH_DOM_ARGUMENTS_H +#define JSONBENCH_DOM_ARGUMENTS_H + +/* Shared adapter plumbing, not a replacement JSON parser. Native libraries + * still validate grammar and build their own DOM. The lexical preflight bounds + * allocations and checks boundaries that some native DOM APIs cannot retain. */ +#include "../config.h" +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct { + unsigned int max_depth, max_args; + int silence; +} dom_settings; + +static void dom_settings_init(dom_settings *settings) { + settings->max_depth = 500; + settings->max_args = 500; + settings->silence = 0; +} + +static int dom_error(char **error, const char *message) { + if (error != NULL) *error = strdup(message); + return 2; +} + +#if HAVE_NEWLOCALE && HAVE_USELOCALE && HAVE_FREELOCALE && HAVE_DUPLOCALE && defined(LC_NUMERIC_MASK) +typedef struct { locale_t current, previous; } dom_locale_guard; + +/* JSON numbers always use '.', regardless of the embedding application's + * process or thread locale. Change only this thread and restore its previous + * locale on every exit; never call the process-wide setlocale here. */ +static int dom_locale_begin(dom_locale_guard *guard, char **error) { + locale_t base = duplocale(uselocale((locale_t)0)); + if (base == (locale_t)0) return dom_error(error, "Unable to duplicate caller numeric locale"); + guard->current = newlocale(LC_NUMERIC_MASK, "C", base); + if (guard->current == (locale_t)0) { + freelocale(base); + return dom_error(error, "Unable to allocate JSON numeric locale"); + } + guard->previous = uselocale(guard->current); + if (guard->previous == (locale_t)0) { + freelocale(guard->current); guard->current = (locale_t)0; + return dom_error(error, "Unable to activate JSON numeric thread locale"); + } + return 0; +} + +static int dom_locale_finish(dom_locale_guard *guard, int result, char **error) { + if (guard->current != (locale_t)0) { + /* Do not free a locale if an unexpected restore failure leaves it + * active. Valid caller-owned previous locales restore successfully. */ + if (uselocale(guard->previous) == (locale_t)0) + return result != 0 ? result : dom_error(error, "Unable to restore numeric thread locale"); + freelocale(guard->current); guard->current = (locale_t)0; + } + return result; +} +#else +typedef struct { int unused; } dom_locale_guard; +static int dom_locale_begin(dom_locale_guard *guard, char **error) { + const struct lconv *current = localeconv(); + (void)guard; + /* A platform without per-thread locales remains usable in an existing + * JSON-compatible numeric locale. Never change its process-wide locale. */ + if (current != NULL && current->decimal_point != NULL && + strcmp(current->decimal_point, ".") == 0) return 0; + return dom_error(error, "Native numeric DOM parsing requires POSIX thread locale support"); +} +static int dom_locale_finish(dom_locale_guard *guard, int result, char **error) { + (void)guard; (void)error; + return result; +} +#endif + +static int dom_set_limit(double value, unsigned int *limit) { + if (limit == NULL || !isfinite(value) || value < 0 || + value > UINT_MAX || floor(value) != value) return -1; + *limit = (unsigned int)value; + return 0; +} + +/* Read regular files and pipes without seek/ftell assumptions. DOM adapters + * necessarily retain document-sized memory, unlike the SAX adapters. */ +static int dom_read_file(const char *filename, char **content, size_t *length, + char **error) { + FILE *file = fopen(filename, "rb"); + size_t used = 0, capacity = 16384; + char *buffer; + if (file == NULL) return dom_error(error, "Unable to open JSON file"); + buffer = (char *)malloc(capacity); + if (buffer == NULL) { + fclose(file); + return dom_error(error, "Unable to allocate JSON input"); + } + for (;;) { + size_t count; + if (used == capacity - 1) { + char *grown; + if (capacity > SIZE_MAX / 2) { + free(buffer); fclose(file); + return dom_error(error, "JSON input size overflow"); + } + capacity *= 2; + grown = (char *)realloc(buffer, capacity); + if (grown == NULL) { + free(buffer); fclose(file); + return dom_error(error, "Unable to allocate JSON input"); + } + buffer = grown; + } + count = fread(buffer + used, 1, capacity - used - 1, file); + used += count; + if (count == 0) { + if (ferror(file)) { + free(buffer); fclose(file); + return dom_error(error, "Unable to read JSON file"); + } + break; + } + } + if (fclose(file) != 0) { + free(buffer); + return dom_error(error, "Unable to close JSON file"); + } + buffer[used] = '\0'; + *content = buffer; *length = used; + return 0; +} + +static int dom_space(unsigned char c) { + return c == ' ' || c == '\t' || c == '\r' || c == '\n'; +} + +static int dom_hex(unsigned char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; +} + +static int dom_code_unit(const char *data, size_t length, size_t *position, + unsigned int *code) { + unsigned int result = 0; + size_t i; + if (length - *position < 4) return 0; + for (i = 0; i < 4; ++i) { + int digit = dom_hex((unsigned char)data[(*position)++]); + if (digit < 0) return 0; + result = result * 16 + (unsigned int)digit; + } + *code = result; + return 1; +} + +/* Strict scalar UTF-8, excluding overlong forms, surrogates and > U+10ffff. */ +static int dom_utf8(const char *data, size_t length, size_t *position) { + unsigned char first = (unsigned char)data[*position]; + size_t count, i; + unsigned int code, minimum; + if (first < 0x80) { ++*position; return 1; } + if (first >= 0xc2 && first <= 0xdf) { count = 2; code = first & 31; minimum = 0x80; } + else if (first >= 0xe0 && first <= 0xef) { count = 3; code = first & 15; minimum = 0x800; } + else if (first >= 0xf0 && first <= 0xf4) { count = 4; code = first & 7; minimum = 0x10000; } + else return 0; + if (length - *position < count) return 0; + for (i = 1; i < count; ++i) { + unsigned char byte = (unsigned char)data[*position + i]; + if ((byte & 0xc0) != 0x80) return 0; + code = code * 64 + (byte & 63); + } + if (code < minimum || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) return 0; + *position += count; + return 1; +} + +enum { DOM_NUL_ALLOWED = 0, DOM_NUL_KEY_FORBIDDEN = 1, DOM_NUL_FORBIDDEN = 2 }; +enum { DOM_NUM_RAW = 0, DOM_NUM_INT64_UINT64 = 1, DOM_NUM_DOUBLE = 2, DOM_NUM_SIGNED = 3 }; +enum { DOM_NOT_NUMBER = 0, DOM_INTEGER = 1, DOM_REAL = 2, DOM_ANY_NUMBER = 3 }; + +typedef struct { + const char *data; + size_t length; + int kind; +} dom_number_span; + +typedef struct { + dom_number_span *spans; + size_t count, capacity; +} dom_numbers; + +static int dom_remember_number(dom_numbers *numbers, const char *data, + size_t length, int kind, char **error) { + dom_number_span *grown; + if (numbers == NULL) return 0; + if (numbers->count == numbers->capacity) { + size_t next = numbers->capacity ? numbers->capacity * 2 : 32; + if (next < numbers->capacity || next > SIZE_MAX / sizeof(*grown)) + return dom_error(error, "Numeric source span allocation overflow"); + grown = (dom_number_span *)realloc(numbers->spans, next * sizeof(*grown)); + if (grown == NULL) return dom_error(error, "Unable to allocate numeric source spans"); + numbers->spans = grown; numbers->capacity = next; + } + numbers->spans[numbers->count].data = data; + numbers->spans[numbers->count].length = length; + numbers->spans[numbers->count++].kind = kind; + return 0; +} + +/* Tokens are checked before native parsing so hostile long strings/numbers + * and deep structures cannot first allocate an unbounded native token/stack. + * No structure acceptance depends on this scan: native parsing follows. */ +static int dom_preflight_impl(const char *data, size_t length, const dom_settings *settings, + int nul_policy, int number_policy, size_t *keys, + dom_numbers *numbers, char **error) { + size_t position = 0, depth = 0, arguments = 0; + *keys = 0; + while (position < length) { + unsigned char c = (unsigned char)data[position]; + if (dom_space(c) || c == ':' || c == ',') { ++position; continue; } + if (c == '{' || c == '[') { + if (++depth > settings->max_depth) return dom_error(error, "JSON depth limit exceeded"); + ++position; continue; + } + if (c == '}' || c == ']') { if (depth > 0) --depth; ++position; continue; } + if (c == '"') { + size_t decoded = 0, end; + int closed = 0, contains_nul = 0; + ++position; + while (position < length) { + c = (unsigned char)data[position++]; + if (c == '"') { closed = 1; break; } + if (c < 0x20) return dom_error(error, "Invalid control character in JSON string"); + if (c == '\\') { + unsigned int code; + if (position == length) return dom_error(error, "Truncated JSON escape"); + c = (unsigned char)data[position++]; + if (c == 'u') { + if (!dom_code_unit(data, length, &position, &code)) return dom_error(error, "Invalid Unicode escape"); + if (code >= 0xd800 && code <= 0xdbff) { + unsigned int low; + if (length - position < 2 || data[position] != '\\' || data[position + 1] != 'u') + return dom_error(error, "Unpaired Unicode surrogate"); + position += 2; + if (!dom_code_unit(data, length, &position, &low) || low < 0xdc00 || low > 0xdfff) + return dom_error(error, "Invalid Unicode surrogate pair"); + decoded += 4; + } else { + if (code >= 0xdc00 && code <= 0xdfff) return dom_error(error, "Unpaired Unicode surrogate"); + if (code == 0) contains_nul = 1; + decoded += code < 0x80 ? 1 : code < 0x800 ? 2 : 3; + } + } else if (c == '"' || c == '\\' || c == '/' || c == 'b' || c == 'f' || c == 'n' || c == 'r' || c == 't') { + ++decoded; + } else return dom_error(error, "Invalid JSON escape"); + } else if (c >= 0x80) { + size_t start = --position; + if (!dom_utf8(data, length, &position)) return dom_error(error, "Invalid UTF-8 in JSON string"); + decoded += position - start; + } else ++decoded; + if (decoded >= JSON_STRING_SIZE) return dom_error(error, "Argument name or value too long"); + } + if (!closed) return dom_error(error, "Truncated JSON string"); + end = position; + while (end < length && dom_space((unsigned char)data[end])) ++end; + if (end < length && data[end] == ':') { + ++*keys; + if (contains_nul && nul_policy != DOM_NUL_ALLOWED) return dom_error(error, "Native DOM cannot preserve NUL in object keys"); + } else { + if (contains_nul && nul_policy == DOM_NUL_FORBIDDEN) return dom_error(error, "Native DOM cannot preserve NUL in string values"); + if (++arguments > settings->max_args) return dom_error(error, "Argument number limit exceeded"); + } + continue; + } + if (c == '-' || (c >= '0' && c <= '9')) { + size_t start = position, digits, token_length; + int negative = c == '-', real = 0; + if (negative) ++position; + digits = position; + if (position == length || data[position] < '0' || data[position] > '9') return dom_error(error, "Invalid JSON number"); + if (data[position] == '0') ++position; + else while (position < length && data[position] >= '0' && data[position] <= '9') ++position; + if (position < length && data[position] == '.') { + size_t fraction = ++position; + real = 1; + while (position < length && data[position] >= '0' && data[position] <= '9') ++position; + if (position == fraction) return dom_error(error, "Invalid JSON fraction"); + } + if (position < length && (data[position] == 'e' || data[position] == 'E')) { + size_t exponent; + real = 1; ++position; + if (position < length && (data[position] == '+' || data[position] == '-')) ++position; + exponent = position; + while (position < length && data[position] >= '0' && data[position] <= '9') ++position; + if (position == exponent) return dom_error(error, "Invalid JSON exponent"); + } + if (position < length && !dom_space((unsigned char)data[position]) && data[position] != ',' && data[position] != ']' && data[position] != '}') + return dom_error(error, "Invalid JSON number separator"); + token_length = position - start; + if (token_length >= JSON_STRING_SIZE) return dom_error(error, "Argument value too long"); + if (!real && number_policy != DOM_NUM_RAW) { + const char *maximum = number_policy == DOM_NUM_DOUBLE ? "9007199254740992" : + negative ? "9223372036854775808" : + number_policy == DOM_NUM_SIGNED ? "9223372036854775807" : "18446744073709551615"; + size_t integer_length = position - digits, max_length = strlen(maximum); + if (integer_length > max_length || (integer_length == max_length && memcmp(data + digits, maximum, max_length) > 0)) + return dom_error(error, "JSON integer is outside the native DOM's lossless range"); + } + if (real && number_policy != DOM_NUM_RAW) { + char number[JSON_STRING_SIZE], *end; + double value; + memcpy(number, data + start, token_length); number[token_length] = '\0'; + errno = 0; value = strtod(number, &end); + if (*end != '\0' || !isfinite(value) || (errno == ERANGE && value == 0.0)) + return dom_error(error, "JSON real is outside the native DOM's finite range"); + } + if (++arguments > settings->max_args) return dom_error(error, "Argument number limit exceeded"); + if (dom_remember_number(numbers, data + start, token_length, + real ? DOM_REAL : DOM_INTEGER, error) != 0) return 2; + continue; + } + if (c == 't' || c == 'f' || c == 'n') { + const char *literal = c == 't' ? "true" : c == 'f' ? "false" : "null"; + size_t count = strlen(literal); + if (length - position < count || memcmp(data + position, literal, count) != 0) return dom_error(error, "Invalid JSON literal"); + position += count; + if (++arguments > settings->max_args) return dom_error(error, "Argument number limit exceeded"); + continue; + } + return dom_error(error, "Unexpected character in JSON input"); + } + return 0; +} + +static inline int dom_preflight(const char *data, size_t length, const dom_settings *settings, + int nul_policy, int number_policy, size_t *keys, char **error) { + dom_locale_guard locale = {0}; + int result; + if (number_policy == DOM_NUM_RAW) + return dom_preflight_impl(data, length, settings, nul_policy, number_policy, keys, NULL, error); + result = dom_locale_begin(&locale, error); + if (result != 0) return result; + result = dom_preflight_impl(data, length, settings, nul_policy, number_policy, keys, NULL, error); + return dom_locale_finish(&locale, result, error); +} + +/* Borrow numeric tokens from the still-live input. Only native parse success + * makes these eligible for output; source-order native traversal checks their + * number kind and total count, including during silent validation. Native + * range checks remain in effect, even though output avoids binary reformatting. */ +static inline int dom_preflight_numbers(const char *data, size_t length, + const dom_settings *settings, int nul_policy, + int number_policy, size_t *keys, + dom_numbers *numbers, char **error) { + dom_locale_guard locale = {0}; + int result; + if (number_policy == DOM_NUM_RAW) + return dom_preflight_impl(data, length, settings, nul_policy, number_policy, keys, numbers, error); + result = dom_locale_begin(&locale, error); + if (result != 0) return result; + result = dom_preflight_impl(data, length, settings, nul_policy, number_policy, keys, numbers, error); + return dom_locale_finish(&locale, result, error); +} + +enum { DOM_SCALAR = 0, DOM_OBJECT = 1, DOM_ARRAY = 2 }; +/* Large/aligned enough for native cursor structs; adapters use memcpy. */ +typedef union { void *pointer; size_t index; void *slots[8]; long double alignment; } dom_cursor; +typedef struct { const char *data; size_t length; char number[128]; int number_kind; } dom_value; +typedef struct { + int (*kind)(const void *node); + int (*child)(const void *node, dom_cursor *cursor, int first, + const void **child, const char **key, size_t *key_length); + int (*scalar)(const void *node, dom_value *value, char **error); +} dom_adapter; + +typedef struct { + const void *node; + dom_cursor cursor; + int first; + size_t prefix_before, key_length; + char *key_before; +} dom_frame; + +static int dom_walk_impl(const void *root, const dom_adapter *adapter, + const dom_settings *settings, size_t *object_keys, + const dom_numbers *numbers, char **error) { + dom_frame *frames = NULL; + size_t depth = 0, capacity = 0, prefix_length = 0, key_length = 0; + unsigned int arguments = 0; + size_t number_position = 0; + char prefix[JSON_STRING_SIZE], key[JSON_STRING_SIZE]; + const void *node = root; + int result = 0, entering = 1; + *object_keys = 0; + for (;;) { + if (entering) { + int kind = adapter->kind(node); + if (kind != DOM_SCALAR) { + dom_frame *frame; + size_t new_length = kind == DOM_ARRAY && depth == 0 ? 5 : + prefix_length + (prefix_length ? 1 : 0) + key_length; + if (depth >= settings->max_depth) { result = dom_error(error, "JSON depth limit exceeded"); break; } + if (new_length >= JSON_STRING_SIZE) { result = dom_error(error, "Argument name too long"); break; } + if (depth == capacity) { + dom_frame *grown; + size_t next = capacity ? capacity * 2 : 16; + if (next < capacity || next > SIZE_MAX / sizeof(*frames)) { result = dom_error(error, "DOM traversal stack overflow"); break; } + grown = (dom_frame *)realloc(frames, next * sizeof(*frames)); + if (grown == NULL) { result = dom_error(error, "Unable to allocate DOM traversal stack"); break; } + frames = grown; capacity = next; + } + frame = &frames[depth]; + memset(frame, 0, sizeof(*frame)); + frame->node = node; frame->first = 1; + frame->prefix_before = prefix_length; frame->key_length = key_length; + frame->key_before = (char *)malloc(key_length + 1); + if (frame->key_before == NULL) { result = dom_error(error, "Unable to allocate DOM traversal key"); break; } + memcpy(frame->key_before, key, key_length); + if (kind == DOM_ARRAY && depth == 0) { + memcpy(prefix, "array", 5); memcpy(key, "array", 5); + prefix_length = key_length = 5; + } else { + if (prefix_length) prefix[prefix_length++] = '.'; + memcpy(prefix + prefix_length, key, key_length); + prefix_length += key_length; + } + ++depth; + } else { + dom_value value; + size_t name_length = prefix_length + (prefix_length ? 1 : 0) + key_length; + memset(&value, 0, sizeof(value)); + if (arguments >= settings->max_args) { result = dom_error(error, "Argument number limit exceeded"); break; } + if (name_length >= JSON_STRING_SIZE) { result = dom_error(error, "Argument name too long"); break; } + result = adapter->scalar(node, &value, error); + if (result != 0) break; + if (numbers != NULL && value.number_kind != DOM_NOT_NUMBER) { + const dom_number_span *span; + if (number_position >= numbers->count) { + result = dom_error(error, "Native DOM has unexpected numeric arguments"); break; + } + span = &numbers->spans[number_position++]; + if (value.number_kind != DOM_ANY_NUMBER && value.number_kind != span->kind) { + result = dom_error(error, "Native DOM numeric type differs from source token"); break; + } + value.data = span->data; value.length = span->length; + } + if (value.length >= JSON_STRING_SIZE) { result = dom_error(error, "Argument value too long"); break; } + ++arguments; + if (!settings->silence) { + int ok = 1; + if (prefix_length) { + ok = fwrite(prefix, 1, prefix_length, stdout) == prefix_length; + ok = (fputc('.', stdout) != EOF) && ok; + } + ok = (fwrite(key, 1, key_length, stdout) == key_length) && ok; + ok = (fwrite(": ", 1, 2, stdout) == 2) && ok; + if (value.length) ok = (fwrite(value.data, 1, value.length, stdout) == value.length) && ok; + ok = (fputc('\n', stdout) != EOF) && ok; + if (!ok) { result = dom_error(error, "Unable to write JSON argument output"); break; } + } + } + } + entering = 0; + while (depth) { + dom_frame *frame = &frames[depth - 1]; + const char *child_key = NULL; + size_t child_key_length = 0; + if (adapter->child(frame->node, &frame->cursor, frame->first, + &node, &child_key, &child_key_length)) { + frame->first = 0; + if (adapter->kind(frame->node) == DOM_OBJECT) { + ++*object_keys; + if (child_key_length >= JSON_STRING_SIZE) { result = dom_error(error, "Argument name too long"); goto done; } + memcpy(key, child_key, child_key_length); key_length = child_key_length; + } + entering = 1; + break; + } + prefix_length = frame->prefix_before; + key_length = frame->key_length; + memcpy(key, frame->key_before, key_length); + free(frame->key_before); + --depth; + } + if (!entering) break; + } +done: + while (depth) free(frames[--depth].key_before); + free(frames); + if (result == 0 && numbers != NULL && number_position != numbers->count) + result = dom_error(error, "Native DOM omitted numeric arguments from source"); + if (result == 0 && !settings->silence && fflush(stdout) != 0) + result = dom_error(error, "Unable to flush JSON argument output"); + return result; +} + +static inline int dom_walk(const void *root, const dom_adapter *adapter, + const dom_settings *settings, size_t *object_keys, char **error) { + return dom_walk_impl(root, adapter, settings, object_keys, NULL, error); +} + +static inline int dom_walk_numbers(const void *root, const dom_adapter *adapter, + const dom_settings *settings, size_t *object_keys, + const dom_numbers *numbers, char **error) { + return dom_walk_impl(root, adapter, settings, object_keys, numbers, error); +} + +#endif diff --git a/src/janssonparser.c b/src/janssonparser.c index ceaac23..e912a37 100644 --- a/src/janssonparser.c +++ b/src/janssonparser.c @@ -1,67 +1,108 @@ #include "janssonparser.h" - #if HAVE_JANSSON - -#include -#include -#include +#include "domarguments.h" #include -struct jansson_parser { - int silence; -}; +struct jansson_parser { dom_settings settings; }; -int jansson_json_init(jansson_parser **parser, char **error_msg) { - assert(parser != NULL); - assert(error_msg != NULL); +static int jansson_kind(const void *node) { + return json_is_object((json_t *)node) ? DOM_OBJECT : json_is_array((json_t *)node) ? DOM_ARRAY : DOM_SCALAR; +} - *parser = (jansson_parser *)calloc(1, sizeof(jansson_parser)); - if (*parser == NULL) { - *error_msg = strdup("Unable to allocate Jansson parser"); - return 1; +static int jansson_child(const void *node, dom_cursor *cursor, int first, + const void **child, const char **key, size_t *key_length) { + json_t *json = (json_t *)node; + if (json_is_array(json)) { + if (first) cursor->index = 0; + if (cursor->index >= json_array_size(json)) return 0; + *child = json_array_get(json, cursor->index++); + } else { + cursor->pointer = first ? json_object_iter(json) : json_object_iter_next(json, cursor->pointer); + if (cursor->pointer == NULL) return 0; + *key = json_object_iter_key(cursor->pointer); + *key_length = json_object_iter_key_len(cursor->pointer); + *child = json_object_iter_value(cursor->pointer); } + return 1; +} - *error_msg = NULL; +static int jansson_scalar(const void *node, dom_value *value, char **error) { + json_t *json = (json_t *)node; + int length; + if (json_is_null(json)) { value->data = ""; value->length = 0; } + else if (json_is_boolean(json)) { + value->data = json_is_true(json) ? "true" : "false"; value->length = strlen(value->data); + } else if (json_is_string(json)) { + value->data = json_string_value(json); value->length = json_string_length(json); + } else if (json_is_integer(json)) { + length = snprintf(value->number, sizeof(value->number), "%" JSON_INTEGER_FORMAT, json_integer_value(json)); + if (length < 0 || (size_t)length >= sizeof(value->number)) return dom_error(error, "Unable to format Jansson integer"); + value->data = value->number; value->length = (size_t)length; + value->number_kind = DOM_INTEGER; + } else if (json_is_real(json)) { + double number = json_real_value(json); + if (!isfinite(number)) return dom_error(error, "Nonfinite Jansson number"); + length = snprintf(value->number, sizeof(value->number), "%.17g", number); + if (length < 0 || (size_t)length >= sizeof(value->number)) return dom_error(error, "Unable to format Jansson real"); + value->data = value->number; value->length = (size_t)length; + value->number_kind = DOM_REAL; + } else return dom_error(error, "Unexpected Jansson scalar type"); return 0; } -int jansson_parse_file(jansson_parser *parser, const char *filename, char **error_msg) { - (void)parser; - if (filename == NULL || error_msg == NULL) { - return -1; - } +static const dom_adapter jansson_adapter = {jansson_kind, jansson_child, jansson_scalar}; - json_error_t jerror; - json_t *root = json_load_file(filename, 0, &jerror); - if (root == NULL) { - *error_msg = strdup(jerror.text); - return 2; - } +int jansson_parse_file(jansson_parser *parser, const char *filename, char **error_msg) { + char *content = NULL; + size_t length = 0, keys = 0; + dom_numbers numbers = {0}; + dom_locale_guard locale = {0}; + json_error_t native_error; + json_t *root = NULL; + int rc; + if (error_msg == NULL) return -1; + *error_msg = NULL; + if (parser == NULL || filename == NULL) return dom_error(error_msg, "Invalid Jansson parser arguments"); + rc = dom_locale_begin(&locale, error_msg); + if (rc != 0) return rc; + rc = dom_read_file(filename, &content, &length, error_msg); + if (rc != 0) goto done; + rc = dom_preflight_numbers(content, length, &parser->settings, DOM_NUL_ALLOWED, + DOM_NUM_SIGNED, &keys, &numbers, error_msg); + if (rc != 0) goto done; + /* Native objects coalesce keys; fail explicitly instead of silently losing + * occurrences. JSON_ALLOW_NUL preserves length-aware decoded strings. */ + root = json_loadb(content, length, JSON_DECODE_ANY | JSON_ALLOW_NUL | JSON_REJECT_DUPLICATES, &native_error); + if (root == NULL) { rc = dom_error(error_msg, native_error.text); goto done; } + rc = dom_walk_numbers(root, &jansson_adapter, &parser->settings, &keys, &numbers, error_msg); + done: + if (root != NULL) json_decref(root); + free(numbers.spans); + free(content); + return dom_locale_finish(&locale, rc, error_msg); +} - json_decref(root); +int jansson_json_init(jansson_parser **parser, char **error_msg) { + if (error_msg == NULL) return -1; + *error_msg = NULL; + if (parser == NULL) return dom_error(error_msg, "Invalid Jansson parser output pointer"); + *parser = (jansson_parser *)calloc(1, sizeof(**parser)); + if (*parser == NULL) return dom_error(error_msg, "Unable to allocate Jansson parser"); + dom_settings_init(&(*parser)->settings); return 0; } -int jansson_set_max_depth(jansson_parser *parser, double max_depth) { - if (parser == NULL) { - return -1; - } - (void)max_depth; - return 1; +int jansson_set_max_depth(jansson_parser *parser, double value) { + return parser == NULL ? -1 : dom_set_limit(value, &parser->settings.max_depth); } -int jansson_set_max_arg_num(jansson_parser *parser, double max_arg_num) { - if (parser == NULL) { - return -1; - } - (void)max_arg_num; - return 1; +int jansson_set_max_arg_num(jansson_parser *parser, double value) { + return parser == NULL ? -1 : dom_set_limit(value, &parser->settings.max_args); } int jansson_set_silence(jansson_parser *parser, int silence) { - if (parser != NULL) { - parser->silence = silence; - } + if (parser == NULL) return -1; + parser->settings.silence = silence != 0; return 0; } diff --git a/src/jsoncparser.c b/src/jsoncparser.c index cb56ad2..65d41bd 100644 --- a/src/jsoncparser.c +++ b/src/jsoncparser.c @@ -1,67 +1,191 @@ #include "jsoncparser.h" - #if HAVE_JSONC - -#include -#include -#include +#include "domarguments.h" #include -#include +#include -struct jsonc_parser { - int silence; -}; +struct jsonc_parser { dom_settings settings; }; -int jsonc_json_init(jsonc_parser **parser, char **error_msg) { - assert(parser != NULL); - assert(error_msg != NULL); +static int jsonc_kind(const void *node) { + enum json_type type = json_object_get_type((struct json_object *)node); + return type == json_type_object ? DOM_OBJECT : type == json_type_array ? DOM_ARRAY : DOM_SCALAR; +} - *parser = (jsonc_parser *)calloc(1, sizeof(jsonc_parser)); - if (*parser == NULL) { - *error_msg = strdup("Unable to allocate JSON-C parser"); - return 1; +static int jsonc_child(const void *node, dom_cursor *cursor, int first, + const void **child, const char **key, size_t *key_length) { + struct json_object *object = (struct json_object *)node; + if (jsonc_kind(node) == DOM_ARRAY) { + if (first) cursor->index = 0; + if (cursor->index >= json_object_array_length(object)) return 0; + *child = json_object_array_get_idx(object, cursor->index++); + } else { + struct lh_entry *entry = first ? lh_table_head(json_object_get_object(object)) : + lh_entry_next((struct lh_entry *)cursor->pointer); + if (entry == NULL) return 0; + cursor->pointer = entry; + *key = (const char *)lh_entry_k(entry); *key_length = strlen(*key); + *child = lh_entry_v(entry); } + return 1; +} - *error_msg = NULL; +static int jsonc_scalar(const void *node, dom_value *value, char **error) { + struct json_object *object = (struct json_object *)node; + switch (json_object_get_type(object)) { + case json_type_null: value->data = ""; value->length = 0; break; + case json_type_boolean: + value->data = json_object_get_boolean(object) ? "true" : "false"; + value->length = strlen(value->data); break; + case json_type_string: + value->data = json_object_get_string(object); + value->length = (size_t)json_object_get_string_len(object); break; + case json_type_int: + case json_type_double: + /* json-c retains the original serialized text for parsed doubles. + * Integer range is checked before its saturating conversion. */ + value->data = json_object_to_json_string_ext(object, JSON_C_TO_STRING_PLAIN); + if (value->data == NULL) return dom_error(error, "Unable to serialize JSON-C number"); + value->length = strlen(value->data); break; + default: return dom_error(error, "Unexpected JSON-C scalar type"); + } + if (json_object_get_type(object) == json_type_int) value->number_kind = DOM_INTEGER; + else if (json_object_get_type(object) == json_type_double) value->number_kind = DOM_REAL; return 0; } -int jsonc_parse_file(jsonc_parser *parser, const char *filename, char **error_msg) { - (void)parser; - if (filename == NULL || error_msg == NULL) { - return -1; +static const dom_adapter jsonc_adapter = {jsonc_kind, jsonc_child, jsonc_scalar}; + +static void jsonc_unicode_escape(char *output, size_t *used, unsigned int code) { + static const char hex[] = "0123456789abcdef"; + output[(*used)++] = '\\'; output[(*used)++] = 'u'; + output[(*used)++] = hex[(code >> 12) & 15]; + output[(*used)++] = hex[(code >> 8) & 15]; + output[(*used)++] = hex[(code >> 4) & 15]; + output[(*used)++] = hex[code & 15]; +} + +/* JSON-C 0.18's strict tokener rejects raw non-ASCII UTF-8 on this platform. + * Preserve strict native grammar checking by converting already verified + * scalar UTF-8 to equivalent JSON Unicode escapes. This is a representation + * conversion only; escaped characters and structural bytes are unchanged. */ +static int jsonc_ascii_input(const char *data, size_t length, char **ascii, + size_t *ascii_length, char **error) { + size_t position = 0, used = 0, output_length = length; + char *output; + while (position < length) { + unsigned char first = (unsigned char)data[position]; + size_t start = position, encoded; + if (first < 0x80) { ++position; continue; } + /* The preflight above guarantees successful scalar decoding. */ + if (!dom_utf8(data, length, &position)) return dom_error(error, "Invalid JSON-C Unicode input"); + encoded = first < 0xf0 ? 6 : 12; + if (output_length > SIZE_MAX - encoded - 1) return dom_error(error, "JSON-C input size overflow"); + output_length += encoded - (position - start); + } + if (output_length == length) { *ascii = NULL; *ascii_length = length; return 0; } + output = (char *)malloc(output_length + 1); + if (output == NULL) return dom_error(error, "Unable to allocate JSON-C Unicode input"); + position = 0; + while (position < length) { + unsigned int code = (unsigned char)data[position++]; + size_t count, i; + if (code < 0x80) { output[used++] = (char)code; continue; } + if (code < 0xe0) { code &= 31; count = 1; } + else if (code < 0xf0) { code &= 15; count = 2; } + else { code &= 7; count = 3; } + for (i = 0; i < count; ++i) code = code * 64 + ((unsigned char)data[position++] & 63); + if (code <= 0xffff) { + jsonc_unicode_escape(output, &used, code); + } else { + unsigned int offset = code - 0x10000; + jsonc_unicode_escape(output, &used, 0xd800 + (offset >> 10)); + jsonc_unicode_escape(output, &used, 0xdc00 + (offset & 1023)); + } } + output[used] = '\0'; + *ascii = output; *ascii_length = used; + return 0; +} - struct json_object *root = json_object_from_file(filename); - if (root == NULL) { - *error_msg = strdup("json_object_from_file failed"); - return 2; +int jsonc_parse_file(jsonc_parser *parser, const char *filename, char **error_msg) { + char *content = NULL, *ascii = NULL; + const char *input; + size_t length = 0, keys = 0, native_keys = 0, end; + dom_numbers numbers = {0}; + dom_locale_guard locale = {0}; + struct json_tokener *tokener = NULL; + struct json_object *root = NULL; + dom_settings check; + int rc; + if (error_msg == NULL) return -1; + *error_msg = NULL; + if (parser == NULL || filename == NULL) return dom_error(error_msg, "Invalid JSON-C parser arguments"); + rc = dom_locale_begin(&locale, error_msg); + if (rc != 0) return rc; + rc = dom_read_file(filename, &content, &length, error_msg); + if (rc != 0) goto done; + rc = dom_preflight_numbers(content, length, &parser->settings, DOM_NUL_KEY_FORBIDDEN, + DOM_NUM_INT64_UINT64, &keys, &numbers, error_msg); + if (rc != 0) goto done; + rc = jsonc_ascii_input(content, length, &ascii, &length, error_msg); + if (rc != 0) goto done; + input = ascii != NULL ? ascii : content; + if (length >= INT_MAX) { + rc = dom_error(error_msg, "JSON-C input exceeds native API range"); goto done; } + /* The native tokener counts scalar slots as well as container frames. + * Bound native state allocation even when a caller sets UINT_MAX. Limits + * above 8192 do not remove this adapter's independent native stack cap. */ + tokener = json_tokener_new_ex((int)(parser->settings.max_depth < JSON_STRING_SIZE ? + parser->settings.max_depth : JSON_STRING_SIZE) + 1); + if (tokener == NULL) { rc = dom_error(error_msg, "Unable to allocate JSON-C tokener"); goto done; } + json_tokener_set_flags(tokener, JSON_TOKENER_STRICT | JSON_TOKENER_VALIDATE_UTF8); + /* Include our sentinel so a root number at physical EOF is finalized. Raw + * NUL inside the actual input was rejected by the lexical boundary guard. */ + root = json_tokener_parse_ex(tokener, input, (int)length + 1); + if (json_tokener_get_error(tokener) != json_tokener_success) { + rc = dom_error(error_msg, json_tokener_error_desc(json_tokener_get_error(tokener))); goto done; + } + end = json_tokener_get_parse_end(tokener); + while (end < length && dom_space((unsigned char)input[end])) ++end; + if (end < length) { rc = dom_error(error_msg, "Unexpected content after JSON document"); goto done; } + /* JSON-C coalesces repeated keys. Check retained vs. input key counts before + * emitting output; reject rather than silently dropping security arguments. */ + check = parser->settings; check.silence = 1; + rc = dom_walk(root, &jsonc_adapter, &check, &native_keys, error_msg); + if (rc != 0) goto done; + if (native_keys != keys) { rc = dom_error(error_msg, "JSON-C DOM cannot preserve repeated object keys"); goto done; } + rc = dom_walk_numbers(root, &jsonc_adapter, &parser->settings, &native_keys, &numbers, error_msg); + done: + if (root != NULL) json_object_put(root); + if (tokener != NULL) json_tokener_free(tokener); + free(numbers.spans); + free(ascii); + free(content); + return dom_locale_finish(&locale, rc, error_msg); +} - json_object_put(root); +int jsonc_json_init(jsonc_parser **parser, char **error_msg) { + if (error_msg == NULL) return -1; + *error_msg = NULL; + if (parser == NULL) return dom_error(error_msg, "Invalid JSON-C parser output pointer"); + *parser = (jsonc_parser *)calloc(1, sizeof(**parser)); + if (*parser == NULL) return dom_error(error_msg, "Unable to allocate JSON-C parser"); + dom_settings_init(&(*parser)->settings); return 0; } -int jsonc_set_max_depth(jsonc_parser *parser, double max_depth) { - if (parser == NULL) { - return -1; - } - (void)max_depth; - return 1; +int jsonc_set_max_depth(jsonc_parser *parser, double value) { + return parser == NULL ? -1 : dom_set_limit(value, &parser->settings.max_depth); } -int jsonc_set_max_arg_num(jsonc_parser *parser, double max_arg_num) { - if (parser == NULL) { - return -1; - } - (void)max_arg_num; - return 1; +int jsonc_set_max_arg_num(jsonc_parser *parser, double value) { + return parser == NULL ? -1 : dom_set_limit(value, &parser->settings.max_args); } int jsonc_set_silence(jsonc_parser *parser, int silence) { - if (parser != NULL) { - parser->silence = silence; - } + if (parser == NULL) return -1; + parser->settings.silence = silence != 0; return 0; } diff --git a/src/jsoncppparser.cpp b/src/jsoncppparser.cpp index dc7c4bd..75731e6 100644 --- a/src/jsoncppparser.cpp +++ b/src/jsoncppparser.cpp @@ -2,77 +2,262 @@ #if HAVE_JSONCPP +#include "pathhandler.h" +#include +#include +#include +#include +#include #include +#include #include -#include +#include +#include #include -#include struct jsoncpp_parser { - int silence; + unsigned int max_depth = 500; + unsigned int max_args = 500; + bool silence = false; }; -extern "C" int jsoncpp_json_init(jsoncpp_parser **parser, char **error_msg) { - if (parser == nullptr || error_msg == nullptr) { - return -1; - } +namespace { - try { - *parser = new jsoncpp_parser{}; - } catch (const std::bad_alloc&) { - *parser = nullptr; - *error_msg = strdup("Unable to allocate JsonCpp parser"); - return 1; +bool valid_limit(double value) { + return std::isfinite(value) && value >= 0 && value <= UINT_MAX && + std::floor(value) == value; +} + +int hex_digit(unsigned char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; +} + +// JsonCpp accepts raw invalid UTF-8 and lone low surrogate escapes. Validate +// these lexical boundaries before constructing its native DOM. The library +// remains responsible for document grammar and decoded values. +void validate_encoding(const std::string& text) { + bool in_string = false; + for (std::size_t i = 0; i < text.size(); ++i) { + const unsigned char c = static_cast(text[i]); + if (c == 0) throw std::invalid_argument("Unexpected NUL in JSON input"); + if (in_string && c < 0x20) + throw std::invalid_argument("Unescaped control character in JSON string"); + if (c >= 0x80) { + unsigned int code = 0, count = 0, minimum = 0; + if (c >= 0xc2 && c <= 0xdf) { code = c & 0x1f; count = 1; minimum = 0x80; } + else if (c >= 0xe0 && c <= 0xef) { code = c & 0x0f; count = 2; minimum = 0x800; } + else if (c >= 0xf0 && c <= 0xf4) { code = c & 0x07; count = 3; minimum = 0x10000; } + else throw std::invalid_argument("Invalid UTF-8 in JSON input"); + if (count > text.size() - i - 1) + throw std::invalid_argument("Truncated UTF-8 in JSON input"); + for (unsigned int n = 0; n < count; ++n) { + const unsigned char next = static_cast(text[++i]); + if ((next & 0xc0) != 0x80) + throw std::invalid_argument("Invalid UTF-8 in JSON input"); + code = (code << 6) | (next & 0x3f); + } + if (code < minimum || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) + throw std::invalid_argument("Invalid UTF-8 code point in JSON input"); + continue; + } + if (c == '"') { in_string = !in_string; continue; } + if (!in_string || c != '\\') continue; + if (++i == text.size()) break; // The reader diagnoses truncation. + if (text[i] != 'u') continue; + auto unit = [&text](std::size_t at) { + if (text.size() - at < 4) + throw std::invalid_argument("Truncated Unicode escape"); + unsigned int code = 0; + for (unsigned int n = 0; n < 4; ++n) { + int digit = hex_digit(static_cast(text[at + n])); + if (digit < 0) throw std::invalid_argument("Invalid Unicode escape"); + code = (code << 4) | static_cast(digit); + } + return code; + }; + const unsigned int first = unit(i + 1); + i += 4; + if (first >= 0xdc00 && first <= 0xdfff) + throw std::invalid_argument("Unpaired low Unicode surrogate"); + if (first < 0xd800 || first > 0xdbff) continue; + if (text.size() - i - 1 < 6 || text[i + 1] != '\\' || text[i + 2] != 'u') + throw std::invalid_argument("Unpaired high Unicode surrogate"); + const unsigned int second = unit(i + 3); + if (second < 0xdc00 || second > 0xdfff) + throw std::invalid_argument("Invalid Unicode surrogate pair"); + i += 6; } +} - *error_msg = nullptr; - return 0; +bool valid_number(const std::string& token) { + std::size_t i = 0; + if (i < token.size() && token[i] == '-') ++i; + if (i == token.size()) return false; + if (token[i] == '0') ++i; + else { + if (token[i] < '1' || token[i] > '9') return false; + while (i < token.size() && token[i] >= '0' && token[i] <= '9') ++i; + } + if (i < token.size() && token[i] == '.') { + const std::size_t start = ++i; + while (i < token.size() && token[i] >= '0' && token[i] <= '9') ++i; + if (i == start) return false; + } + if (i < token.size() && (token[i] == 'e' || token[i] == 'E')) { + ++i; + if (i < token.size() && (token[i] == '+' || token[i] == '-')) ++i; + const std::size_t start = i; + while (i < token.size() && token[i] >= '0' && token[i] <= '9') ++i; + if (i == start) return false; + } + return i == token.size(); } -extern "C" int jsoncpp_parse_file(jsoncpp_parser *parser, const char *filename, char **error_msg) { - (void)parser; - if (filename == nullptr || error_msg == nullptr) { - return -1; +class Arguments { + const jsoncpp_parser& settings_; + const std::string& input_; + PathHandler path_; + unsigned int count_ = 0; + + void check_path() const { + if (path_.m_error) throw std::length_error("Argument name too long"); } - std::ifstream input(filename); - if (!input.is_open()) { - *error_msg = strdup("Unable to open JSON file"); - return 2; + void emit(const std::string& value) { + const std::string name = path_.name(); + check_path(); + if (value.size() >= JSON_STRING_SIZE) + throw std::length_error("Argument value too long"); + if (count_ >= settings_.max_args) + throw std::length_error("Argument number limit exceeded"); + ++count_; + if (settings_.silence) return; + if (std::fwrite(name.data(), 1, name.size(), stdout) != name.size() || + std::fwrite(": ", 1, 2, stdout) != 2 || + std::fwrite(value.data(), 1, value.size(), stdout) != value.size() || + std::fputc('\n', stdout) == EOF) + throw std::runtime_error("Unable to write JSON argument output"); } - Json::CharReaderBuilder builder; - Json::Value root; - std::string errs; - const bool ok = Json::parseFromStream(builder, input, &root, &errs); - if (!ok) { - *error_msg = strdup(errs.c_str()); - return 2; + public: + Arguments(const jsoncpp_parser& settings, const std::string& input) + : settings_(settings), input_(input) {} + + void visit(const Json::Value& value, unsigned int depth = 0) { + if (value.isObject() || value.isArray()) { + if (depth >= settings_.max_depth) + throw std::length_error("JSON depth limit exceeded"); + if (value.isArray()) path_.startArraySpecial(); + else path_.push(); + check_path(); + if (value.isArray()) { + for (const auto& child : value) visit(child, depth + 1); + } else { + // JsonCpp stores objects in key order. Source offsets restore + // input order for unique keys; duplicate keys are rejected. + auto names = value.getMemberNames(); + std::sort(names.begin(), names.end(), [&value](const std::string& a, const std::string& b) { + return value[a].getOffsetStart() < value[b].getOffsetStart(); + }); + for (const auto& key : names) { + path_.setKey(key.data(), key.size()); + check_path(); + visit(value[key], depth + 1); + } + } + path_.pop(); + return; + } + if (value.isNull()) emit(""); + else if (value.isBool()) emit(value.asBool() ? "true" : "false"); + else if (value.isString()) emit(value.asString()); + else if (value.isNumeric()) { + const auto start = value.getOffsetStart(); + const auto end = value.getOffsetLimit(); + if (start < 0 || end < start || static_cast(end) > input_.size()) + throw std::runtime_error("Invalid JsonCpp numeric source offsets"); + const std::string token = input_.substr(static_cast(start), + static_cast(end - start)); + if (!valid_number(token)) throw std::invalid_argument("Invalid JSON number"); + // Preserve the validated spelling, including integers beyond 64 + // bits, rather than converting its DOM double back to text. + emit(token); + } else throw std::invalid_argument("Unsupported JSON value"); } +}; - return 0; +} // namespace + +extern "C" int jsoncpp_json_init(jsoncpp_parser **parser, char **error_msg) { + if (!parser || !error_msg) return -1; + *parser = nullptr; + *error_msg = nullptr; + try { *parser = new jsoncpp_parser{}; return 0; } + catch (const std::exception& e) { *error_msg = strdup(e.what()); return 1; } + catch (...) { *error_msg = strdup("Unable to initialize JsonCpp parser"); return 1; } } -extern "C" int jsoncpp_set_max_depth(jsoncpp_parser *parser, double max_depth) { - if (parser == nullptr) { - return -1; +extern "C" int jsoncpp_parse_file(jsoncpp_parser *parser, const char *filename, char **error_msg) { + if (error_msg) *error_msg = nullptr; + if (!parser || !filename || !error_msg) return -1; + try { + std::ifstream file(filename, std::ios::binary); + if (!file.is_open()) throw std::runtime_error("Unable to open JSON file"); + const std::string input{std::istreambuf_iterator(file), std::istreambuf_iterator()}; + if (file.bad()) throw std::runtime_error("Unable to read JSON file"); + validate_encoding(input); + Json::CharReaderBuilder builder; + builder["collectComments"] = false; + builder["allowComments"] = false; + builder["allowTrailingCommas"] = false; + builder["strictRoot"] = false; + builder["allowDroppedNullPlaceholders"] = false; + builder["allowNumericKeys"] = false; + builder["allowSingleQuotes"] = false; + builder["failIfExtra"] = true; + builder["rejectDupKeys"] = true; + builder["allowSpecialFloats"] = false; + builder["skipBom"] = false; + // Bound native recursive parsing even if a caller requests UINT_MAX. + // Traversal enforces the exact configured container depth below. + builder["stackLimit"] = static_cast( + std::min(parser->max_depth, 1023) + 1); + std::unique_ptr reader(builder.newCharReader()); + Json::Value root; + std::string errors; + if (!reader->parse(input.data(), input.data() + input.size(), &root, &errors)) + throw std::invalid_argument(errors); + Arguments(*parser, input).visit(root); + if (!parser->silence && std::fflush(stdout) == EOF) + throw std::runtime_error("Unable to write JSON argument output"); + return 0; + } catch (const std::exception& e) { + *error_msg = strdup(e.what()); + return 2; + } catch (...) { + *error_msg = strdup("JsonCpp parsing failed"); + return 2; } - (void)max_depth; - return 1; } -extern "C" int jsoncpp_set_max_arg_num(jsoncpp_parser *parser, double max_arg_num) { - if (parser == nullptr) { - return -1; - } - (void)max_arg_num; - return 1; +extern "C" int jsoncpp_set_max_depth(jsoncpp_parser *parser, double value) { + if (!parser || !valid_limit(value)) return -1; + parser->max_depth = static_cast(value); + return 0; +} + +extern "C" int jsoncpp_set_max_arg_num(jsoncpp_parser *parser, double value) { + if (!parser || !valid_limit(value)) return -1; + parser->max_args = static_cast(value); + return 0; } extern "C" int jsoncpp_set_silence(jsoncpp_parser *parser, int silence) { - if (parser != nullptr) { - parser->silence = silence; - } + if (!parser) return -1; + parser->silence = silence != 0; return 0; } diff --git a/src/simdjsonparser.cpp b/src/simdjsonparser.cpp index c979590..360dceb 100644 --- a/src/simdjsonparser.cpp +++ b/src/simdjsonparser.cpp @@ -2,152 +2,206 @@ #if HAVE_SIMDJSON +#include "pathhandler.h" #include +#include +#include +#include #include -#include +#include +#include +#include +#include struct simdjson_parser { - int silence; + unsigned int max_depth = 500; + unsigned int max_args = 500; + bool silence = false; simdjson::ondemand::parser parser; - simdjson_parser() : silence(0), parser() {} }; -static int consume_value(simdjson::ondemand::value value) { - simdjson::ondemand::json_type type; - auto type_result = value.type(); - if (type_result.error()) { - return 1; +namespace { + +bool valid_limit(double value) { + return std::isfinite(value) && value >= 0 && value <= UINT_MAX && + std::floor(value) == value; +} + +void check(simdjson::error_code error) { + if (error) throw std::invalid_argument(simdjson::error_message(error)); +} + +class Arguments { + const simdjson_parser& settings_; + PathHandler path_; + unsigned int count_ = 0; + + void check_path() const { + if (path_.m_error) throw std::length_error("Argument name too long"); } - type = type_result.value_unsafe(); - switch (type) { - case simdjson::ondemand::json_type::object: { - auto obj_result = value.get_object(); - if (obj_result.error()) { - return 1; - } - auto obj = obj_result.value_unsafe(); - for (auto field_result : obj) { - if (field_result.error()) { - return 1; - } - auto field = field_result.value_unsafe(); - auto key_result = field.unescaped_key(false); - if (key_result.error()) { - return 1; + void check_depth(unsigned int depth) const { + if (depth >= settings_.max_depth) + throw std::length_error("JSON depth limit exceeded"); + // Recursive visitor frames are larger in sanitizer builds. Keep an + // independent adapter ceiling so raising the user limit cannot exhaust + // the process stack before reaching the library's own state limit. + if (depth >= 512) + throw std::length_error("simdjson adapter JSON depth limit (512) exceeded"); + // On-Demand's parser has its own finite state stack. Calling get_array + // or get_object beyond it can assert in development builds; stop before + // accessing that stack even when the caller requests a larger limit. + if (static_cast(depth) + 1 >= settings_.parser.max_depth()) + throw std::length_error("simdjson native JSON depth limit exceeded"); + } + + void emit(std::string_view value) { + const std::string name = path_.name(); + check_path(); + if (value.size() >= JSON_STRING_SIZE) + throw std::length_error("Argument value too long"); + if (count_ >= settings_.max_args) + throw std::length_error("Argument number limit exceeded"); + ++count_; + if (settings_.silence) return; + if (std::fwrite(name.data(), 1, name.size(), stdout) != name.size() || + std::fwrite(": ", 1, 2, stdout) != 2 || + std::fwrite(value.data(), 1, value.size(), stdout) != value.size() || + std::fputc('\n', stdout) == EOF) + throw std::runtime_error("Unable to write JSON argument output"); + } + + public: + explicit Arguments(const simdjson_parser& settings) : settings_(settings) {} + + // The document overload is essential for scalar roots: converting a root + // document to ondemand::value only works for arrays/objects in simdjson. + template + void visit(Value& value, unsigned int depth = 0) { + simdjson::ondemand::json_type type = simdjson::ondemand::json_type::null; + check(value.type().get(type)); + switch (type) { + case simdjson::ondemand::json_type::object: { + check_depth(depth); + path_.push(); + check_path(); + simdjson::ondemand::object object; + check(value.get_object().get(object)); + for (auto field_result : object) { + simdjson::ondemand::field field; + check(std::move(field_result).get(field)); + std::string_view key; + check(field.unescaped_key(false).get(key)); + path_.setKey(key.data(), key.size()); + check_path(); + auto child = field.value(); + visit(child, depth + 1); } - auto child = field.value(); - int rc = consume_value(child); - if (rc != 0) { - return rc; + path_.pop(); + break; + } + case simdjson::ondemand::json_type::array: { + check_depth(depth); + path_.startArraySpecial(); + check_path(); + simdjson::ondemand::array array; + check(value.get_array().get(array)); + for (auto child_result : array) { + simdjson::ondemand::value child; + check(std::move(child_result).get(child)); + visit(child, depth + 1); } + path_.pop(); + break; } - return 0; - } - case simdjson::ondemand::json_type::array: { - auto arr_result = value.get_array(); - if (arr_result.error()) { - return 1; + case simdjson::ondemand::json_type::number: { + std::string_view raw; + // Capturing the public raw token preserves integer precision + // and decimal spelling; get_number still validates and consumes + // every numeric value. Unsupported ranges fail explicitly. + check(value.raw_json().get(raw)); + while (!raw.empty() && (raw.back() == ' ' || raw.back() == '\t' || + raw.back() == '\r' || raw.back() == '\n')) raw.remove_suffix(1); + simdjson::ondemand::number number; + check(value.get_number().get(number)); + emit(raw); + break; } - auto arr = arr_result.value_unsafe(); - for (auto elem_result : arr) { - if (elem_result.error()) { - return 1; - } - auto elem = elem_result.value_unsafe(); - int rc = consume_value(elem); - if (rc != 0) { - return rc; - } + case simdjson::ondemand::json_type::string: { + std::string_view text; + check(value.get_string(false).get(text)); + emit(text); + break; + } + case simdjson::ondemand::json_type::boolean: { + bool flag = false; + check(value.get_bool().get(flag)); + emit(flag ? "true" : "false"); + break; + } + case simdjson::ondemand::json_type::null: { + bool is_null = false; + check(value.is_null().get(is_null)); + if (!is_null) throw std::invalid_argument("Invalid JSON null"); + emit(""); + break; } - return 0; - } - case simdjson::ondemand::json_type::number: { - auto number_result = value.get_number(); - return number_result.error() ? 1 : 0; - } - case simdjson::ondemand::json_type::string: { - auto str_result = value.get_string(); - return str_result.error() ? 1 : 0; - } - case simdjson::ondemand::json_type::boolean: { - auto bool_result = value.get_bool(); - return bool_result.error() ? 1 : 0; - } - case simdjson::ondemand::json_type::null: { - auto null_result = value.is_null(); - return null_result.error() ? 1 : 0; } } +}; - return 1; -} +} // namespace extern "C" int simdjson_json_init(simdjson_parser **parser, char **error_msg) { - if (parser == nullptr || error_msg == nullptr) { - return -1; - } - - try { - *parser = new simdjson_parser{}; - } catch (const std::bad_alloc&) { - *parser = nullptr; - *error_msg = strdup("Unable to allocate simdjson parser"); - return 1; - } - + if (!parser || !error_msg) return -1; + *parser = nullptr; *error_msg = nullptr; - return 0; + try { *parser = new simdjson_parser{}; return 0; } + catch (const std::exception& e) { *error_msg = strdup(e.what()); return 1; } + catch (...) { *error_msg = strdup("Unable to initialize simdjson parser"); return 1; } } extern "C" int simdjson_parse_file(simdjson_parser *parser, const char *filename, char **error_msg) { - if (parser == nullptr || filename == nullptr || error_msg == nullptr) { - return -1; - } - - auto json_result = simdjson::padded_string::load(filename); - if (json_result.error()) { - *error_msg = strdup(simdjson::error_message(json_result.error())); - return 2; - } - - auto &json = json_result.value_unsafe(); - auto doc_result = parser->parser.iterate(json); - if (doc_result.error()) { - *error_msg = strdup(simdjson::error_message(doc_result.error())); + if (error_msg) *error_msg = nullptr; + if (!parser || !filename || !error_msg) return -1; + try { + // On-Demand avoids a DOM but needs a complete padded input buffer. + auto json_result = simdjson::padded_string::load(filename); + check(json_result.error()); + auto& json = json_result.value_unsafe(); + auto result = parser->parser.iterate(json); + check(result.error()); + auto& document = result.value_unsafe(); + Arguments(*parser).visit(document); + if (!document.at_end()) + throw std::invalid_argument("Unexpected content after JSON document"); + if (!parser->silence && std::fflush(stdout) == EOF) + throw std::runtime_error("Unable to write JSON argument output"); + return 0; + } catch (const std::exception& e) { + *error_msg = strdup(e.what()); return 2; - } - - auto &doc = doc_result.value_unsafe(); - int rc = consume_value(doc); - if (rc != 0) { - *error_msg = strdup("simdjson parse/iteration failed"); + } catch (...) { + *error_msg = strdup("simdjson parsing failed"); return 2; } - - return 0; } -extern "C" int simdjson_set_max_depth(simdjson_parser *parser, double max_depth) { - if (parser == nullptr) { - return -1; - } - (void)max_depth; - return 1; +extern "C" int simdjson_set_max_depth(simdjson_parser *parser, double value) { + if (!parser || !valid_limit(value)) return -1; + parser->max_depth = static_cast(value); + return 0; } -extern "C" int simdjson_set_max_arg_num(simdjson_parser *parser, double max_arg_num) { - if (parser == nullptr) { - return -1; - } - (void)max_arg_num; - return 1; +extern "C" int simdjson_set_max_arg_num(simdjson_parser *parser, double value) { + if (!parser || !valid_limit(value)) return -1; + parser->max_args = static_cast(value); + return 0; } extern "C" int simdjson_set_silence(simdjson_parser *parser, int silence) { - if (parser != nullptr) { - parser->silence = silence; - } + if (!parser) return -1; + parser->silence = silence != 0; return 0; } diff --git a/src/yyjsonparser.c b/src/yyjsonparser.c index 4384335..15ef504 100644 --- a/src/yyjsonparser.c +++ b/src/yyjsonparser.c @@ -1,118 +1,104 @@ #include "yyjsonparser.h" - #if HAVE_YYJSON - -#include -#include -#include -#include +#include "domarguments.h" #include -struct yyjson_parser { - int silence; -}; - -static int read_whole_file(const char *filename, char **content, size_t *length, char **error_msg) { - FILE *fp = fopen(filename, "rb"); - if (fp == NULL) { - *error_msg = strdup("Unable to open JSON file"); - return 1; - } - - if (fseek(fp, 0, SEEK_END) != 0) { - fclose(fp); - *error_msg = strdup("Unable to seek JSON file"); - return 1; - } - - long len = ftell(fp); - if (len < 0) { - fclose(fp); - *error_msg = strdup("Unable to measure JSON file"); - return 1; - } - rewind(fp); - - char *buffer = (char *)malloc((size_t)len + 1U); - if (buffer == NULL) { - fclose(fp); - *error_msg = strdup("Unable to allocate JSON buffer"); - return 1; - } +struct yyjson_parser { dom_settings settings; }; - size_t read_len = fread(buffer, 1, (size_t)len, fp); - fclose(fp); - if (read_len != (size_t)len) { - free(buffer); - *error_msg = strdup("Unable to read entire JSON file"); - return 1; - } - - buffer[len] = '\0'; - *content = buffer; - *length = (size_t)len; - return 0; +static int yyjson_kind(const void *node) { + return yyjson_is_obj((yyjson_val *)node) ? DOM_OBJECT : yyjson_is_arr((yyjson_val *)node) ? DOM_ARRAY : DOM_SCALAR; } -int yyjson_json_init(yyjson_parser **parser, char **error_msg) { - assert(parser != NULL); - assert(error_msg != NULL); - - *parser = (yyjson_parser *)calloc(1, sizeof(yyjson_parser)); - if (*parser == NULL) { - *error_msg = strdup("Unable to allocate yyjson parser"); - return 1; +static int yyjson_child(const void *node, dom_cursor *cursor, int first, + const void **child, const char **key, size_t *key_length) { + yyjson_val *json = (yyjson_val *)node; + if (yyjson_is_arr(json)) { + yyjson_arr_iter iter; + if (first) iter = yyjson_arr_iter_with(json); + else memcpy(&iter, cursor, sizeof(iter)); + *child = yyjson_arr_iter_next(&iter); + if (*child == NULL) return 0; + memcpy(cursor, &iter, sizeof(iter)); + } else { + yyjson_obj_iter iter; + yyjson_val *native_key; + if (first) iter = yyjson_obj_iter_with(json); + else memcpy(&iter, cursor, sizeof(iter)); + native_key = yyjson_obj_iter_next(&iter); + if (native_key == NULL) return 0; + *child = yyjson_obj_iter_get_val(native_key); + *key = yyjson_get_str(native_key); *key_length = yyjson_get_len(native_key); + memcpy(cursor, &iter, sizeof(iter)); } + return 1; +} - *error_msg = NULL; +static int yyjson_scalar(const void *node, dom_value *value, char **error) { + yyjson_val *json = (yyjson_val *)node; + if (yyjson_is_null(json)) { value->data = ""; value->length = 0; } + else if (yyjson_is_bool(json)) { + value->data = yyjson_get_bool(json) ? "true" : "false"; value->length = strlen(value->data); + } else if (yyjson_is_str(json)) { + value->data = yyjson_get_str(json); value->length = yyjson_get_len(json); + } else if (yyjson_is_raw(json)) { + value->data = yyjson_get_raw(json); value->length = yyjson_get_len(json); + } else return dom_error(error, "Unexpected non-lossless yyjson scalar type"); return 0; } -int yyjson_parse_file(yyjson_parser *parser, const char *filename, char **error_msg) { - (void)parser; - if (filename == NULL || error_msg == NULL) { - return -1; - } +static const dom_adapter yyjson_adapter = {yyjson_kind, yyjson_child, yyjson_scalar}; +int yyjson_parse_file(yyjson_parser *parser, const char *filename, char **error_msg) { char *content = NULL; - size_t length = 0; - int rc = read_whole_file(filename, &content, &length, error_msg); - if (rc != 0) { - return 2; - } - - yyjson_doc *doc = yyjson_read(content, length, 0); - free(content); - + size_t length = 0, keys = 0; + yyjson_read_err native_error; + yyjson_doc *doc = NULL; + int rc; + if (error_msg == NULL) return -1; + *error_msg = NULL; + if (parser == NULL || filename == NULL) return dom_error(error_msg, "Invalid yyjson parser arguments"); + rc = dom_read_file(filename, &content, &length, error_msg); + if (rc != 0) return rc; + rc = dom_preflight(content, length, &parser->settings, DOM_NUL_ALLOWED, + DOM_NUM_RAW, &keys, error_msg); + if (rc != 0) goto done; + /* Public native option preserves arbitrary JSON number lexemes and the + * native object iterator retains repeated keys in original order. */ + doc = yyjson_read_opts(content, length, YYJSON_READ_NUMBER_AS_RAW, NULL, &native_error); if (doc == NULL) { - *error_msg = strdup("yyjson_read failed"); - return 2; + char message[256]; + snprintf(message, sizeof(message), "yyjson parse error at byte %zu: %s", native_error.pos, + native_error.msg != NULL ? native_error.msg : "unknown error"); + rc = dom_error(error_msg, message); goto done; } - + rc = dom_walk(yyjson_doc_get_root(doc), &yyjson_adapter, &parser->settings, &keys, error_msg); + done: yyjson_doc_free(doc); + free(content); + return rc; +} + +int yyjson_json_init(yyjson_parser **parser, char **error_msg) { + if (error_msg == NULL) return -1; + *error_msg = NULL; + if (parser == NULL) return dom_error(error_msg, "Invalid yyjson parser output pointer"); + *parser = (yyjson_parser *)calloc(1, sizeof(**parser)); + if (*parser == NULL) return dom_error(error_msg, "Unable to allocate yyjson parser"); + dom_settings_init(&(*parser)->settings); return 0; } -int yyjson_set_max_depth(yyjson_parser *parser, double max_depth) { - if (parser == NULL) { - return -1; - } - (void)max_depth; - return 1; +int yyjson_set_max_depth(yyjson_parser *parser, double value) { + return parser == NULL ? -1 : dom_set_limit(value, &parser->settings.max_depth); } -int yyjson_set_max_arg_num(yyjson_parser *parser, double max_arg_num) { - if (parser == NULL) { - return -1; - } - (void)max_arg_num; - return 1; +int yyjson_set_max_arg_num(yyjson_parser *parser, double value) { + return parser == NULL ? -1 : dom_set_limit(value, &parser->settings.max_args); } int yyjson_set_silence(yyjson_parser *parser, int silence) { - if (parser != NULL) { - parser->silence = silence; - } + if (parser == NULL) return -1; + parser->settings.silence = silence != 0; return 0; } diff --git a/tests/Makefile.am b/tests/Makefile.am new file mode 100644 index 0000000..22d5b77 --- /dev/null +++ b/tests/Makefile.am @@ -0,0 +1,46 @@ +# Keep direct API contracts in the project's standard Automake test runner. +# Disabled adapters compile their guarded stubs; their harnesses return 77, +# which Automake reports as SKIP instead of a successful compatibility check. +AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src +AM_CFLAGS = -Wall +AM_CXXFLAGS = -Wall -std=c++17 + +check_PROGRAMS = pathhandler_tests jsoncons_api_tests yajl_api_tests \ + sax_adapter_api_tests dom_adapter_api_tests cpp_adapter_api_tests +TESTS = $(check_PROGRAMS) + +pathhandler_tests_SOURCES = pathhandler_tests.cpp + +jsoncons_api_tests_SOURCES = jsoncons_api_tests.cpp ../src/jsonconsparser.cpp +jsoncons_api_tests_CPPFLAGS = $(AM_CPPFLAGS) + +yajl_api_tests_SOURCES = yajl_api_tests.cpp ../src/yajlparser.c +yajl_api_tests_CPPFLAGS = $(AM_CPPFLAGS) $(YAJL_CFLAGS) +yajl_api_tests_LDFLAGS = $(YAJL_LDFLAGS) +yajl_api_tests_LDADD = $(YAJL_LDADD) + +sax_adapter_api_tests_SOURCES = sax_adapter_api_tests.cpp \ + ../src/rjparser.cpp ../src/nlparser.cpp +sax_adapter_api_tests_CPPFLAGS = $(AM_CPPFLAGS) + +dom_adapter_api_tests_SOURCES = dom_adapter_api_tests.c \ + ../src/jsoncparser.c ../src/janssonparser.c \ + ../src/cjsonparser.c ../src/yyjsonparser.c +dom_adapter_api_tests_CPPFLAGS = $(AM_CPPFLAGS) $(JSONC_CFLAGS) \ + $(JANSSON_CFLAGS) $(CJSON_CFLAGS) $(YYJSON_CFLAGS) +dom_adapter_api_tests_LDFLAGS = $(JANSSON_LDFLAGS) $(JSONC_LDFLAGS) \ + $(CJSON_LDFLAGS) $(YYJSON_LDFLAGS) +# Jansson must precede JSON-C for overlapping unversioned object APIs. +dom_adapter_api_tests_LDADD = $(JANSSON_LDADD) $(JSONC_LDADD) \ + $(CJSON_LDADD) $(YYJSON_LDADD) -lm + +cpp_adapter_api_tests_SOURCES = cpp_adapter_api_tests.cpp \ + ../src/jsoncppparser.cpp ../src/simdjsonparser.cpp +cpp_adapter_api_tests_CPPFLAGS = $(AM_CPPFLAGS) $(JSONCPP_CFLAGS) $(SIMDJSON_CFLAGS) +cpp_adapter_api_tests_LDFLAGS = $(JSONCPP_LDFLAGS) $(SIMDJSON_LDFLAGS) +cpp_adapter_api_tests_LDADD = $(JSONCPP_LDADD) $(SIMDJSON_LDADD) + +EXTRA_DIST = README.md run_tests.sh regression_tests.py \ + benchmark_jsoncons.py benchmark_rusage.c \ + test01.json test02.json test03.json test04.json \ + test05.json test06.json 512KB.json diff --git a/tests/README.md b/tests/README.md index 877cbea..9be4514 100644 --- a/tests/README.md +++ b/tests/README.md @@ -6,6 +6,7 @@ commands, then run: ```sh tests/run_tests.sh # every available engine, the same required matrix tests/run_tests.sh JSONCONS # one engine; still compare with YAJL if available +make check # six direct API/path harnesses, optional engines respected ``` The existing shell entry point uses `regression_tests.py`, a Python 3 standard @@ -29,9 +30,11 @@ empty containers do not. A parse-only adapter that emits no arguments **fails** output tests. An adapter that explicitly warns that a particular limit is unsupported can skip that limit check; it still runs parsing, output, numeric and malformed-input tests. -Unavailable YAJL causes a clearly labelled comparison skip. Single-engine runs -never compare the engine with itself. Only byte equality after checking the -expected outputs is reported as compatibility. Known differences are separately +Unavailable YAJL causes a clearly labelled comparison skip. Cross-engine checks +require both engines to be available. YAJL's existing +fixture rechecks are labelled repeatability, rather than compatibility with +itself. Only byte equality after checking the expected outputs is reported as +compatibility. Known differences are separately reported as `DIFF`, not as compatibility passes. Full-engine runs expose existing adapter defects and return nonzero when any required assertion fails. @@ -39,11 +42,11 @@ The required numeric contract preserves exact integer precision (including values outside signed/unsigned 64-bit ranges) and decimal/exponent precision. jsoncons' public lossless-number API normalizes integer `-0` to `0` and removes the optional exponent `+`; those two differences have explicit expected results. -Other engines are checked against the original numerical spelling. Known YAJL -path-restoration, embedded-NUL and root-number-at-EOF differences are measured -separately; when YAJL itself is selected it must still satisfy the intended safe -path/output contract and fails those affected tests. No output-wide normalization -is applied to comparisons. The helper removes only the CLI timing record at its +Other engines are checked against the original numerical spelling, except that +nlohmann's native integer event also has the explicit -0-to-0 expected result. +The YAJL follow-up fixes path restoration, embedded NUL and root numbers at EOF; +those cases now require exact equality. No output-wide normalization is applied +to comparisons. The helper removes only the CLI timing record at its defined position, preserving real blank lines and `Time` argument data. To test another built executable or reproduce baseline failures: @@ -76,3 +79,20 @@ compiler's standard paths. Add `-fsanitize=address,undefined -fno-omit-frame-pointer` to this command to run the harness with sanitizers. The API harness returns nonzero on failure and returns 77 if configured without JSONCONS; that is unavailable coverage, not a passing test. + +`make check` builds and runs all six committed harnesses: PathHandler, jsoncons, +YAJL, the RapidJSON/nlohmann SAX pair, the four C DOM adapters, and the +JsonCpp/simdjson pair. They test C API zero limits and rejected setters, caller +diagnostic ownership, reuse after errors, exact output and streaming boundaries. +The C DOM harness also checks decimal-comma process/thread locales when one is +installed; that subcheck reports an explicit skip otherwise. The C++ harness +preserves the simdjson safety ceiling regression: 512 containers accepted with +an enlarged configured limit, 513/999/2000 rejected diagnostically. + +These API tests target the configured source tree, independently of a +`JSONBENCH_BINARY` override. Missing engines return 77 and Automake reports +SKIP; enabled compilation or test failures remain failures. CLI required tests +continue to run against every available engine and intentionally fail on +unsupported numeric/duplicate/NUL contracts. See +[`docs/adapter-stack.md`](../docs/adapter-stack.md) for the measured matrix and +remaining native-library limitations. diff --git a/tests/benchmark_jsoncons.py b/tests/benchmark_jsoncons.py index 64ae20f..6f8206d 100644 --- a/tests/benchmark_jsoncons.py +++ b/tests/benchmark_jsoncons.py @@ -167,7 +167,7 @@ def main(): parser.add_argument("--binary", type=Path, default=REPOSITORY / "src/jsonbench") parser.add_argument("--baseline", type=Path, help="preserved executable containing the original DOM implementation") parser.add_argument("--config", type=Path, default=REPOSITORY / "config.h", - help="configured header used to build --binary, for YAJL's file buffer limit") + help="configured header used to build --binary, recorded with the measurements") parser.add_argument("--samples", type=positive_int, default=5) parser.add_argument("--warmups", type=positive_int, default=1) parser.add_argument("--sizes", type=size_bytes, nargs="+", default=None) diff --git a/tests/cpp_adapter_api_tests.cpp b/tests/cpp_adapter_api_tests.cpp new file mode 100644 index 0000000..e2ad290 --- /dev/null +++ b/tests/cpp_adapter_api_tests.cpp @@ -0,0 +1,344 @@ +// File-based C API contracts for the native JsonCpp DOM and simdjson On-Demand +// adapters. These complement the common CLI matrix with zero limits, reuse, +// numeric capability differences, and bounded traversal with enlarged limits. +#include "../src/jsoncppparser.h" +#include "../src/simdjsonparser.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if HAVE_JSONCPP || HAVE_SIMDJSON +namespace { + +class Input { + std::string path_; + public: + explicit Input(const std::string& text) { + char name[] = "/tmp/jsonbench-cpp-api-XXXXXX"; + const int fd = mkstemp(name); + if (fd < 0) throw std::runtime_error("mkstemp failed"); + path_ = name; + std::size_t offset = 0; + while (offset < text.size()) { + const ssize_t count = write(fd, text.data() + offset, text.size() - offset); + if (count < 0 && errno == EINTR) continue; + if (count <= 0) { + close(fd); unlink(path_.c_str()); + throw std::runtime_error("temporary input write failed"); + } + offset += static_cast(count); + } + if (close(fd) != 0) { + unlink(path_.c_str()); + throw std::runtime_error("temporary input close failed"); + } + } + ~Input() { unlink(path_.c_str()); } + Input(const Input&) = delete; + Input& operator=(const Input&) = delete; + const char* path() const { return path_.c_str(); } +}; + +class Capture { + FILE* file_ = nullptr; + int saved_ = -1; + public: + Capture() { + if (std::fflush(stdout) != 0) throw std::runtime_error("stdout flush failed"); + file_ = std::tmpfile(); + if (!file_) throw std::runtime_error("output temporary file failed"); + saved_ = dup(STDOUT_FILENO); + if (saved_ < 0 || dup2(fileno(file_), STDOUT_FILENO) < 0) { + if (saved_ >= 0) close(saved_); + std::fclose(file_); + throw std::runtime_error("stdout capture failed"); + } + } + ~Capture() { + std::fflush(stdout); + dup2(saved_, STDOUT_FILENO); + close(saved_); + std::fclose(file_); + } + Capture(const Capture&) = delete; + Capture& operator=(const Capture&) = delete; + std::string read() { + if (std::fflush(stdout) != 0 || std::fseek(file_, 0, SEEK_SET) != 0) + throw std::runtime_error("output rewind failed"); + std::string output; + char buffer[1024]; + std::size_t count; + while ((count = std::fread(buffer, 1, sizeof buffer, file_)) != 0) + output.append(buffer, count); + if (std::ferror(file_)) throw std::runtime_error("output read failed"); + return output; + } +}; + +// The incoming diagnostic belongs to the caller. Successful parsing resets +// the slot without freeing it; failed parsing supplies a free()-owned message. +class ErrorSlot { + char* old_; + public: + char* value; + ErrorSlot() : old_(static_cast(std::malloc(1))), value(old_) { + if (!old_) throw std::bad_alloc(); + *old_ = '\0'; + } + ~ErrorSlot() { + if (value != old_) std::free(value); + std::free(old_); + } + ErrorSlot(const ErrorSlot&) = delete; + ErrorSlot& operator=(const ErrorSlot&) = delete; + bool replaced() const { return value != old_; } +}; + +#define FILE_API(Type, Prefix, Label, DuplicateKeys, WideNumbers, SafeDepth) \ +struct Type { \ + using Handle = Prefix##_parser; \ + static const char* name() { return Label; } \ + static constexpr bool duplicates = DuplicateKeys; \ + static constexpr bool wide_numbers = WideNumbers; \ + static constexpr unsigned int safe_depth = SafeDepth; \ + static int init(Handle** p, char** e) { return Prefix##_json_init(p, e); } \ + static int file(Handle* p, const char* f, char** e) { return Prefix##_parse_file(p, f, e); } \ + static int depth(Handle* p, double n) { return Prefix##_set_max_depth(p, n); } \ + static int arguments(Handle* p, double n) { return Prefix##_set_max_arg_num(p, n); } \ + static int silence(Handle* p, int n) { return Prefix##_set_silence(p, n); } \ + static int cleanup(Handle* p) { return Prefix##_json_cleanup(p); } \ +}; + +#if HAVE_JSONCPP +FILE_API(JsonCppApi, jsoncpp, "JsonCpp", false, true, 999) +#endif +#if HAVE_SIMDJSON +FILE_API(SimdJsonApi, simdjson, "simdjson", true, false, 512) +#endif +#undef FILE_API + +template +class Parser { + public: + typename Api::Handle* value = nullptr; + Parser() { + ErrorSlot error; + if (Api::init(&value, &error.value) != 0 || !value || error.value) { + Api::cleanup(value); + throw std::runtime_error(std::string(Api::name()) + " initialization failed"); + } + } + ~Parser() { Api::cleanup(value); } + Parser(const Parser&) = delete; + Parser& operator=(const Parser&) = delete; +}; + +struct Result { + int status; + bool replaced; + bool null_error; + std::string error; + std::string output; +}; + +template +Result parse_path(typename Api::Handle* parser, const char* path) { + ErrorSlot error; + Capture capture; + const int status = Api::file(parser, path, &error.value); + return {status, error.replaced(), !error.value, + error.value ? error.value : "", capture.read()}; +} + +template +Result parse(typename Api::Handle* parser, const std::string& text) { + Input input(text); + return parse_path(parser, input.path()); +} + +class Checks { + unsigned int passed_ = 0; + unsigned int failed_ = 0; + public: + std::string engine; + void check(const std::string& name, bool ok) { + if (ok) ++passed_; + else { + ++failed_; + std::fprintf(stderr, "FAIL: %s: %s\n", engine.c_str(), name.c_str()); + } + } + void success(const std::string& name, const Result& result, const std::string& expected = "") { + check(name + ": success and reset diagnostic", + result.status == 0 && result.replaced && result.null_error); + check(name + ": exact output", result.output == expected); + if (result.status != 0) std::fprintf(stderr, " diagnostic: %s\n", result.error.c_str()); + } + void failure(const std::string& name, const Result& result, const std::string& diagnostic = "") { + check(name + ": failure and owned diagnostic", result.status != 0 && + result.replaced && !result.null_error && !result.error.empty()); + if (!diagnostic.empty()) + check(name + ": diagnostic identifies limit", result.error.find(diagnostic) != std::string::npos); + } + int finish() const { + std::printf("C++ adapter C APIs: %u checks passed, %u failed\n", passed_, failed_); + return failed_ ? 1 : 0; + } +}; + +template +void invalid_arguments(Checks& checks) { + typename Api::Handle* handle = nullptr; + ErrorSlot error; + checks.check("null initialization slots", Api::init(nullptr, &error.value) != 0 && + Api::init(&handle, nullptr) != 0 && !handle && Api::init(nullptr, nullptr) != 0); + checks.check("null setters and cleanup", Api::depth(nullptr, 1) != 0 && + Api::arguments(nullptr, 1) != 0 && Api::silence(nullptr, 1) != 0 && Api::cleanup(nullptr) == 0); + Parser parser; + Input input("true"); + const Result null_parser = parse_path(nullptr, input.path()); + const Result null_filename = parse_path(parser.value, nullptr); + checks.check("invalid parser/filename reset diagnostics and emit nothing", + null_parser.status != 0 && null_parser.replaced && null_parser.null_error && null_parser.output.empty() && + null_filename.status != 0 && null_filename.replaced && null_filename.null_error && null_filename.output.empty()); + checks.check("null diagnostic slot rejected", Api::file(parser.value, input.path(), nullptr) != 0); + checks.failure("missing file", parse_path(parser.value, (std::string(input.path()) + ".missing").c_str())); + checks.success("reuse after invalid API calls", parse(parser.value, "true"), ": true\n"); +} + +template +void limits(Checks& checks) { + Parser parser; + checks.check("configure two containers and arguments", Api::depth(parser.value, 2) == 0 && + Api::arguments(parser.value, 2) == 0 && Api::silence(parser.value, 1) == 0); + const std::vector invalid = {-1, -0.5, 0.5, 1.5, + std::numeric_limits::quiet_NaN(), std::numeric_limits::infinity(), + -std::numeric_limits::infinity(), static_cast(UINT_MAX) + 1}; + for (std::size_t i = 0; i < invalid.size(); ++i) { + const std::string label = "invalid setter " + std::to_string(i); + checks.check(label, Api::depth(parser.value, invalid[i]) != 0 && Api::arguments(parser.value, invalid[i]) != 0); + checks.success(label + " preserves exact limits", parse(parser.value, "{\"a\":[true,null]}")); + checks.failure(label + " preserves depth ceiling", parse(parser.value, "[[[]]]")); + checks.failure(label + " preserves count ceiling", parse(parser.value, "[true,null,false]")); + } + checks.check("zero depth accepted", Api::depth(parser.value, -0.0) == 0); + for (const char* value : {"true", "null", "42", "\"value\""}) + checks.success(std::string("root scalar at depth zero ") + value, parse(parser.value, value)); + for (const char* value : {"{}", "[]"}) + checks.failure(std::string("container at depth zero ") + value, parse(parser.value, value), "depth"); + checks.check("zero count accepted", Api::depth(parser.value, 3) == 0 && Api::arguments(parser.value, 0) == 0); + checks.success("empty containers at count zero", parse(parser.value, "{\"a\":[{},[]]}")); + for (const char* value : {"true", "false", "null", "0", "\"\""}) + checks.failure(std::string("scalar at count zero ") + value, parse(parser.value, value), "Argument number"); + checks.check("restore exact argument limit", Api::arguments(parser.value, 3) == 0); + checks.success("all scalar kinds counted exactly", parse(parser.value, "[[[null,true,false]]]")); + checks.failure("four scalar leaves exceed count", parse(parser.value, "[null,true,false,0]")); +} + +template +void reuse_and_numbers(Checks& checks) { + Parser parser; + checks.success("source order and structural path restoration", parse(parser.value, + "{\"z\":[{\"b\":true},null],\"a.b\":{\"c\":2},\"after\":false}"), + "z.z.b: true\nz.z: \na.b.c: 2\nafter: false\n"); + const char binary[] = "a\0b: x\0y\n"; + checks.success("embedded NUL name and value", parse(parser.value, "{\"a\\u0000b\":\"x\\u0000y\"}"), + std::string(binary, sizeof binary - 1)); + for (const char* invalid : {"[true,]", "1e+", "\"\\udc00\""}) { + checks.failure("syntax rejection", parse(parser.value, invalid)); + checks.success("clean state after syntax failure", parse(parser.value, "{\"fresh\":false}"), "fresh: false\n"); + } + const std::string finite = "{\"min\":-9223372036854775808,\"max\":9223372036854775807," + "\"u\":18446744073709551615,\"d\":0.123456789012345678901234567890,\"e\":1.2300e+03,\"n\":-0}"; + checks.success("lossless numeric spelling", parse(parser.value, finite), + "min: -9223372036854775808\nmax: 9223372036854775807\nu: 18446744073709551615\n" + "d: 0.123456789012345678901234567890\ne: 1.2300e+03\nn: -0\n"); + for (const char* number : {"18446744073709551616", "-9223372036854775809", "1e9999"}) { + if (Api::wide_numbers) + checks.success("raw numeric range retained", parse(parser.value, number), std::string(": ") + number + '\n'); + else checks.failure("unsupported native numeric range rejected", parse(parser.value, number)); + } + const Result duplicate = parse(parser.value, "{\"k\":1,\"k\":2}"); + if (Api::duplicates) checks.success("duplicate keys preserved", duplicate, "k: 1\nk: 2\n"); + else checks.failure("duplicate keys explicitly rejected", duplicate, "Duplicate key"); + checks.check("nonzero silence accepted", Api::silence(parser.value, -7) == 0); + checks.success("silence suppresses output", parse(parser.value, "{\"quiet\":true}")); + checks.failure("silence retains syntax validation", parse(parser.value, "{} []")); + checks.failure("silence rejects trailing raw NUL", parse(parser.value, std::string("{}\0", 3))); + checks.failure("oversized token rejected", parse(parser.value, "\"" + std::string(JSON_STRING_SIZE, 'x') + "\"")); + checks.success("reuse after native and resource failures", parse(parser.value, "[true,null]")); +} + +template +void large_depths(Checks& checks) { + Parser parser; + checks.check("enlarged integral limits accepted", Api::depth(parser.value, UINT_MAX) == 0 && + Api::arguments(parser.value, UINT_MAX) == 0); + for (bool silent : {false, true}) { + checks.check("set deep traversal mode", Api::silence(parser.value, silent) == 0); + for (unsigned int depth : {500u, 501u, 512u, 513u, 999u, 2000u}) { + for (bool leaf : {false, true}) { + const std::string text = std::string(depth, '[') + (leaf ? "true" : "") + std::string(depth, ']'); + const std::string label = "depth " + std::to_string(depth) + + (leaf ? " boolean leaf" : " empty arrays") + (silent ? " silent" : " output"); + if (depth <= Api::safe_depth) { + std::string expected; + if (leaf && !silent) { + expected = "array"; + for (unsigned int i = 0; i < depth; ++i) expected += ".array"; + expected += ": true\n"; + } + checks.success(label, parse(parser.value, text), expected); + } else checks.failure(label, parse(parser.value, text), Api::duplicates ? "depth" : "stackLimit"); + } + } + } + checks.check("restore default depth and one argument", Api::depth(parser.value, 500) == 0 && + Api::arguments(parser.value, 1) == 0 && Api::silence(parser.value, 1) == 0); + checks.success("default depth exact", parse(parser.value, std::string(500, '[') + "true" + std::string(500, ']'))); + checks.failure("default depth exceeded", parse(parser.value, std::string(501, '[') + "true" + std::string(501, ']'))); + checks.failure("boolean leaf consumes count", parse(parser.value, "[true,false]")); + checks.success("parser reusable after deep failures", parse(parser.value, "true")); +} + +template +void run(Checks& checks) { + checks.engine = Api::name(); + invalid_arguments(checks); + limits(checks); + reuse_and_numbers(checks); + large_depths(checks); +} + +} // namespace +#endif + +int main() { +#if !HAVE_JSONCPP && !HAVE_SIMDJSON + std::puts("SKIP C++ adapter C APIs: neither JsonCpp nor simdjson enabled"); + return 77; +#else + try { + Checks checks; +#if HAVE_JSONCPP + run(checks); +#endif +#if HAVE_SIMDJSON + run(checks); +#endif + return checks.finish(); + } catch (const std::exception& error) { + std::fprintf(stderr, "C++ adapter C API test harness failed: %s\n", error.what()); + return 1; + } +#endif +} diff --git a/tests/dom_adapter_api_tests.c b/tests/dom_adapter_api_tests.c new file mode 100644 index 0000000..3f99c50 --- /dev/null +++ b/tests/dom_adapter_api_tests.c @@ -0,0 +1,174 @@ +#include "../config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "../src/jsoncparser.h" +#include "../src/janssonparser.h" +#include "../src/cjsonparser.h" +#include "../src/yyjsonparser.h" +static unsigned checks; +static char directory[] = "/tmp/jsonbench-dom-api-XXXXXX"; +static char valid[512], invalid[512], native_invalid[512], empty[512], root[512], decimal[512], missing[512]; +static void cleanup_fixtures(void) { + unlink(valid); unlink(invalid); unlink(native_invalid); unlink(empty); unlink(root); unlink(decimal); + rmdir(directory); +} +static void fixture(char *path, size_t capacity, const char *name, const char *content) { + FILE *file; + int length = snprintf(path, capacity, "%s/%s.json", directory, name); + if (length < 0 || (size_t)length >= capacity) exit(1); + file = fopen(path, "wb"); + if (file == NULL || fwrite(content, 1, strlen(content), file) != strlen(content) || fclose(file) != 0) exit(1); +} + +#define CHECK(condition) do { if (!(condition)) { fprintf(stderr,"FAIL line %d: %s\n",__LINE__,#condition); exit(1); } ++checks; } while (0) +#define RELEASE_ERROR() do { free(error); error = NULL; } while (0) +#define ENGINE(prefix) do { \ + prefix##_parser *parser = NULL; char *error = NULL; \ + CHECK(prefix##_json_init(NULL, &error) != 0 && error != NULL); RELEASE_ERROR(); \ + CHECK(prefix##_json_init(&parser, NULL) != 0 && parser == NULL); \ + CHECK(prefix##_set_max_depth(NULL, 1) != 0); \ + CHECK(prefix##_set_max_arg_num(NULL, 1) != 0); \ + CHECK(prefix##_set_silence(NULL, 1) != 0); \ + CHECK(prefix##_parse_file(NULL, valid, &error) != 0 && error != NULL); RELEASE_ERROR(); \ + CHECK(prefix##_json_cleanup(NULL) == 0); \ + CHECK(prefix##_json_init(&parser, &error) == 0 && parser != NULL && error == NULL); \ + double invalid_limits[] = {-1.0, 1.5, INFINITY, -INFINITY, NAN, 4294967296.0}; \ + for (unsigned i=0;idecimal_point, ",") == 0); \ + CHECK(strcmp(setlocale(LC_NUMERIC, NULL), global_before) == 0); \ + CHECK(prefix##_parse_file(parser, invalid, &error) != 0 && error != NULL); RELEASE_ERROR(); \ + CHECK(uselocale((locale_t)0) == before && strcmp(localeconv()->decimal_point, ",") == 0); \ + CHECK(prefix##_parse_file(parser, native_invalid, &error) != 0 && error != NULL); RELEASE_ERROR(); \ + CHECK(uselocale((locale_t)0) == before && strcmp(localeconv()->decimal_point, ",") == 0); \ + CHECK(prefix##_parse_file(parser, missing, &error) != 0 && error != NULL); RELEASE_ERROR(); \ + CHECK(uselocale((locale_t)0) == before && strcmp(localeconv()->decimal_point, ",") == 0); \ + FILE *capture = tmpfile(); int saved_stdout = dup(STDOUT_FILENO); \ + CHECK(capture != NULL && saved_stdout >= 0 && fflush(stdout) == 0); \ + CHECK(dup2(fileno(capture), STDOUT_FILENO) >= 0); \ + CHECK(prefix##_set_silence(parser, 0) == 0); \ + int parse_result = prefix##_parse_file(parser, decimal, &error); \ + CHECK(fflush(stdout) == 0 && dup2(saved_stdout, STDOUT_FILENO) >= 0); close(saved_stdout); \ + CHECK(parse_result == 0 && error == NULL); \ + CHECK(uselocale((locale_t)0) == before && strcmp(localeconv()->decimal_point, ",") == 0); \ + CHECK(strcmp(setlocale(LC_NUMERIC, NULL), global_before) == 0); \ + rewind(capture); char output[128] = {0}; size_t count = fread(output, 1, sizeof(output), capture); \ + const char *expected = "a: 1.25\nb: 1e-3\nneg: -0\n"; \ + CHECK(count == strlen(expected) && memcmp(output, expected, count) == 0); fclose(capture); \ + CHECK(prefix##_json_cleanup(parser) == 0); free(global_before); \ +} while (0) + +static void locale_engines(void) { +#if HAVE_JSONC + LOCALE_ENGINE(jsonc); +#endif +#if HAVE_JANSSON + LOCALE_ENGINE(jansson); +#endif +#if HAVE_CJSON + LOCALE_ENGINE(cjson); +#endif +#if HAVE_YYJSON + LOCALE_ENGINE(yyjson); +#endif +} + +static void locale_checks(void) { + static const char *names[] = {"de_DE.UTF-8", "de_DE.utf8", "fr_FR.UTF-8", "fr_FR.utf8"}; + char *original = strdup(setlocale(LC_NUMERIC, NULL)); + const char *selected = NULL; + locale_t caller_locale, previous; + CHECK(original != NULL); + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); ++i) { + if (setlocale(LC_NUMERIC, names[i]) != NULL && strcmp(localeconv()->decimal_point, ",") == 0) { + selected = names[i]; break; + } + } + if (selected == NULL) { + CHECK(setlocale(LC_NUMERIC, original) != NULL); + free(original); + puts("SKIP numeric locale checks: no installed decimal-comma locale"); + return; + } + /* Global comma locale, then independently selected per-thread comma locale. + * Production adapters must leave both caller settings untouched. */ + locale_engines(); + CHECK(setlocale(LC_NUMERIC, "C") != NULL); + caller_locale = newlocale(LC_NUMERIC_MASK, selected, (locale_t)0); + CHECK(caller_locale != (locale_t)0); + previous = uselocale(caller_locale); + CHECK(previous != (locale_t)0); + locale_engines(); + CHECK(uselocale(previous) == caller_locale); + freelocale(caller_locale); + CHECK(setlocale(LC_NUMERIC, original) != NULL); + free(original); +} +#else +static void locale_checks(void) { + puts("SKIP numeric locale checks: POSIX thread locales unavailable"); +} +#endif + +int main(void) { +#if !HAVE_JSONC && !HAVE_JANSSON && !HAVE_CJSON && !HAVE_YYJSON + puts("SKIP DOM C API checks: no DOM engines enabled"); + return 77; +#endif + CHECK(mkdtemp(directory) != NULL); + CHECK(atexit(cleanup_fixtures) == 0); + fixture(valid, sizeof(valid), "valid", "{\"x\":[1,true,null,{\"k\":\"value\"}]}"); + fixture(invalid, sizeof(invalid), "invalid", "{\"x\":\"\\q\"}"); + fixture(native_invalid, sizeof(native_invalid), "native-invalid", "{\"a\":1,}"); + fixture(empty, sizeof(empty), "empty", "{}"); + fixture(root, sizeof(root), "root", "42"); + fixture(decimal, sizeof(decimal), "decimal", "{\"a\":1.25,\"b\":1e-3,\"neg\":-0}"); + CHECK(snprintf(missing, sizeof(missing), "%s/missing.json", directory) > 0); +#if HAVE_JSONC + ENGINE(jsonc); +#endif +#if HAVE_JANSSON + ENGINE(jansson); +#endif +#if HAVE_CJSON + ENGINE(cjson); +#endif +#if HAVE_YYJSON + ENGINE(yyjson); +#endif + locale_checks(); + printf("%u direct DOM C API checks passed\n", checks); + return 0; +} diff --git a/tests/regression_tests.py b/tests/regression_tests.py index 278a9b9..46abe7c 100755 --- a/tests/regression_tests.py +++ b/tests/regression_tests.py @@ -89,6 +89,9 @@ def utf8(text): Case("embedded-nul", b'{"a\\u0000b":"x\\u0000y"}', b"a\0b: x\0y\n", 1), Case("timing-like-value", b'{"Time":"0.000000000 sec","blank":"\\n"}', b"Time: 0.000000000 sec\nblank: \n\n", 2), + Case("numeric-token-order", + b'{"12":"34","word":"1e+20","list":[11,0.25,{"n":1.20,"s":"-1"},null,true]}', + b"12: 34\nword: 1e+20\nlist.list: 11\nlist.list: 0.25\nlist.list.n: 1.20\nlist.list.s: -1\nlist.list: \nlist.list: true\n", 8), Case("integer-boundaries", b'{"min":-9223372036854775808,"max":9223372036854775807,"u":18446744073709551615,"beyond":18446744073709551616,"negative":-9223372036854775809}', b"min: -9223372036854775808\nmax: 9223372036854775807\nu: 18446744073709551615\nbeyond: 18446744073709551616\nnegative: -9223372036854775809\n", 5), @@ -247,7 +250,8 @@ def baseline(self): path = TESTS_DIR / fixture self.success(engine, f"existing valid parse/silent: {fixture}", path, silent=True) if fixture in reference: - self.success(engine, f"existing argument output vs YAJL: {fixture}", path, reference[fixture]) + label = "existing argument output repeatability" if engine == "YAJL" else "existing argument output vs YAJL" + self.success(engine, f"{label}: {fixture}", path, reference[fixture]) else: self.report("SKIP", f"[{engine}] output comparison: {fixture} (YAJL unavailable; compatibility untested)") self.rejected(engine, "reject existing invalid fixture: test05.json", TESTS_DIR / "test05.json") @@ -359,8 +363,10 @@ def large_document(self, engine): def operational(self, engine): self.rejected(engine, "missing input file diagnostic", self.temp / "does-not-exist.json") - path = self.fixture("iterations", b'{"x":1,"x":2}') - self.success(engine, "three timed iterations print arguments once", path, b"x: 1\nx: 2\n", ["-n", "3", "-a", "2"]) + # Repeated-key behavior has its own mandatory golden case. Unique keys + # here isolate state reset/output count across timed iterations. + path = self.fixture("iterations", b'{"x":1,"y":2}') + self.success(engine, "three timed iterations print arguments once", path, b"x: 1\ny: 2\n", ["-n", "3", "-a", "2"]) self.success(engine, "three silent iterations emit no arguments", path, options=["-n", "3", "-a", "2"], silent=True) self.limit_rejected(engine, "timed iterations retain argument limit", path, ["-n", "3", "-a", "1"]) for option in ("-d", "-a"): From 3d1f65f2de1de828e0f6584109bfb1c7a8c02d61 Mon Sep 17 00:00:00 2001 From: jens Steinbacher Date: Tue, 13 Oct 2026 17:10:37 +0000 Subject: [PATCH 05/18] Use typed RAII and specific exception handling in streaming adapters --- src/nlparser.cpp | 90 ++++++------ src/rjparser.cpp | 130 +++++++++-------- tests/sax_adapter_api_tests.cpp | 247 ++++++++------------------------ 3 files changed, 180 insertions(+), 287 deletions(-) diff --git a/src/nlparser.cpp b/src/nlparser.cpp index a83deaa..7925a3f 100644 --- a/src/nlparser.cpp +++ b/src/nlparser.cpp @@ -28,6 +28,11 @@ namespace { using nljson = nlohmann::json; +class NlohmannInputError final : public std::runtime_error { + public: + using std::runtime_error::runtime_error; +}; + // The public SAX API still materializes individual strings and number tokens. // Guard their raw size before the lexer can allocate an unbounded token. Six // source bytes per decoded byte allow legal \uXXXX escapes at the value limit. @@ -48,7 +53,7 @@ class CheckedInputBuffer final : public std::streambuf { } void check(unsigned char c) { - if (c == 0) throw std::invalid_argument("Invalid raw NUL in JSON input"); + if (c == 0) throw NlohmannInputError("Invalid raw NUL in JSON input"); if (in_string_) { if (!escaped_ && c == '"') { in_string_ = false; @@ -56,7 +61,7 @@ class CheckedInputBuffer final : public std::streambuf { return; } if (++token_length_ > static_cast(JSON_STRING_SIZE) * 6) - throw std::length_error("JSON string token too long"); + throw NlohmannInputError("JSON string token too long"); if (escaped_) escaped_ = false; else if (c == '\\') escaped_ = true; return; @@ -64,7 +69,7 @@ class CheckedInputBuffer final : public std::streambuf { if (in_number_) { if (number_character(c)) { if (++token_length_ >= JSON_STRING_SIZE) - throw std::length_error("JSON number token too long"); + throw NlohmannInputError("JSON number token too long"); return; } in_number_ = false; @@ -83,7 +88,7 @@ class CheckedInputBuffer final : public std::streambuf { std::size_t count; if (file_) { count = std::fread(buffer_.data(), 1, buffer_.size(), file_); - if (std::ferror(file_)) throw std::runtime_error("Error reading JSON file"); + if (std::ferror(file_)) throw NlohmannInputError("Error reading JSON file"); } else { count = remaining_ < buffer_.size() ? remaining_ : buffer_.size(); if (count != 0) std::memcpy(buffer_.data(), memory_, count); @@ -138,7 +143,6 @@ class NLSAXHandler final : public nlohmann::json_sax { const std::string name = path_.name(); if (path_.m_error) return fail("Argument name too long"); if (value.size() >= JSON_STRING_SIZE) return fail("Argument value too long"); - // Unlimited argument mode must not wrap the counter on enormous input. ++arguments_; if (settings_.silence) return true; if (std::fwrite(name.data(), 1, name.size(), stdout) != name.size() || @@ -199,23 +203,39 @@ struct FileCloser { void operator()(std::FILE* file) const noexcept { std::fclose(file); } }; -int parse(nl_parser* parser, CheckedInputBuffer& source, char** error_msg) { +int parse(const nl_parser* parser, CheckedInputBuffer& source, char** error_msg) { + // A fresh visitor keeps failed or completed documents from carrying + // depth, argument counts, errors or paths into a subsequent parse. + NLSAXHandler handler(*parser); + std::istream input(&source); + const bool ok = nljson::sax_parse(input, &handler, + nljson::input_format_t::json, true, false); + if (!ok) return fail(error_msg, handler.error().empty() ? + "NlohmannJSON parsing failed" : handler.error().c_str()); + if (!parser->silence && std::fflush(stdout) != 0) + return fail(error_msg, "Unable to write JSON argument output"); + return 0; +} + +// Specific handlers preserve diagnostics while the final guard protects the +// C interface from unexpected exceptions originating in third-party code. +template +int at_c_boundary(Operation&& operation, char** error_msg, int failure_status = 2) noexcept { try { - // A fresh visitor keeps failed or completed documents from carrying - // depth, argument counts, errors or paths into a subsequent parse. - NLSAXHandler handler(*parser); - std::istream input(&source); - const bool ok = nljson::sax_parse(input, &handler, - nljson::input_format_t::json, true, false); - if (!ok) return fail(error_msg, handler.error().empty() ? - "NlohmannJSON parsing failed" : handler.error().c_str()); - if (!parser->silence && std::fflush(stdout) != 0) - return fail(error_msg, "Unable to write JSON argument output"); - return 0; - } catch (const std::exception& error) { - return fail(error_msg, error.what()); + return operation(); + } catch (const NlohmannInputError& error) { + return fail(error_msg, error.what(), failure_status); + } catch (const nljson::exception& error) { + return fail(error_msg, error.what(), failure_status); + } catch (const std::bad_alloc& error) { + return fail(error_msg, error.what(), failure_status); + } catch (const std::length_error& error) { + return fail(error_msg, error.what(), failure_status); + } catch (const std::ios_base::failure& error) { + return fail(error_msg, error.what(), failure_status); } catch (...) { - return fail(error_msg, "Unexpected NlohmannJSON parser failure"); + // Never allow an unknown third-party exception to cross the C ABI. + return fail(error_msg, "Unexpected NlohmannJSON parser failure", failure_status); } } @@ -225,18 +245,14 @@ extern "C" int nl_json_init(nl_parser** parser, char** error_msg) { if (error_msg) *error_msg = nullptr; if (parser) *parser = nullptr; if (!parser || !error_msg) return -1; - try { - *parser = new nl_parser; + return at_c_boundary([&] { + *parser = std::make_unique().release(); return 0; - } catch (const std::exception& error) { - return fail(error_msg, error.what(), -1); - } catch (...) { - return fail(error_msg, "Unable to initialize NlohmannJSON parser", -1); - } + }, error_msg, -1); } extern "C" int nl_json_cleanup(nl_parser* parser) { - delete parser; + const std::unique_ptr owned(parser); return 0; } @@ -245,30 +261,22 @@ extern "C" int nl_parse_buffer(nl_parser* parser, const char* buffer, if (error_msg) *error_msg = nullptr; if (!parser || !buffer || !error_msg) return fail(error_msg, "Invalid NlohmannJSON parser arguments", -1); - try { + return at_c_boundary([&] { CheckedInputBuffer source(buffer, length); return parse(parser, source, error_msg); - } catch (const std::exception& error) { - return fail(error_msg, error.what()); - } catch (...) { - return fail(error_msg, "Unexpected NlohmannJSON parser failure"); - } + }, error_msg); } extern "C" int nl_parse_file(nl_parser* parser, const char* filename, char** error_msg) { if (error_msg) *error_msg = nullptr; if (!parser || !filename || !error_msg) return fail(error_msg, "Invalid NlohmannJSON parser arguments", -1); - try { - std::unique_ptr file(std::fopen(filename, "rb")); + return at_c_boundary([&] { + const std::unique_ptr file(std::fopen(filename, "rb")); if (!file) return fail(error_msg, "Unable to open JSON file", -1); CheckedInputBuffer source(file.get()); return parse(parser, source, error_msg); - } catch (const std::exception& error) { - return fail(error_msg, error.what()); - } catch (...) { - return fail(error_msg, "Unexpected NlohmannJSON parser failure"); - } + }, error_msg); } extern "C" int nl_set_max_depth(nl_parser* parser, double value) { diff --git a/src/rjparser.cpp b/src/rjparser.cpp index 6e72fd3..8f07da0 100644 --- a/src/rjparser.cpp +++ b/src/rjparser.cpp @@ -4,14 +4,16 @@ #include "pathhandler.h" +#include +#include #include #include -#include #include #include #include #include #include +#include #include #include @@ -26,27 +28,35 @@ struct rj_parser { namespace { +class RapidJsonInputError final : public std::runtime_error { + public: + using std::runtime_error::runtime_error; +}; + // RapidJSON keeps one decoded token and its iterative structural stack here. // A checked allocator stops hostile unterminated strings/numbers from growing // document-sized storage before a SAX callback can enforce the token limit. class TemporaryAllocator { public: - static const bool kNeedFree = true; - void* Malloc(std::size_t size) { return Realloc(nullptr, 0, size); } - void* Realloc(void* data, std::size_t, std::size_t size) { + static constexpr bool kNeedFree = true; + char* Malloc(std::size_t size) const { return Realloc(nullptr, 0, size); } + char* Realloc(char* data, std::size_t old_size, std::size_t size) const { constexpr std::size_t budget = static_cast(JSON_STRING_SIZE) * 4 + 16384; if (size > budget) - throw std::length_error("RapidJSON token/state allocation limit exceeded"); - if (size == 0) { - std::free(data); - return nullptr; - } - void* result = std::realloc(data, size); - if (result == nullptr) throw std::bad_alloc(); - return result; + throw RapidJsonInputError("RapidJSON token/state allocation limit exceeded"); + // RapidJSON 1.1's GenericReader stack passes char* storage to Realloc + // and Free. A char array supplies alignment for its scalar state values. + // Allocate first: an exception leaves the original stack allocation live. + auto replacement = size == 0 ? nullptr : std::make_unique(size); + if (data != nullptr && size != 0) + std::copy_n(data, std::min(old_size, size), replacement.get()); + const std::unique_ptr original(data); + return replacement.release(); + } + static void Free(char* data) noexcept { + const std::unique_ptr allocation(data); } - static void Free(void* data) { std::free(data); } }; // RapidJSON uses zero both for its synthetic EOF sentinel and for an input @@ -65,7 +75,7 @@ class CheckedStream { const std::size_t offset = source_.Tell(); source_.Take(); if (source_.Tell() != offset) - throw std::invalid_argument("Unexpected NUL byte at offset " + + throw RapidJsonInputError("Unexpected NUL byte at offset " + std::to_string(offset)); } return value; @@ -76,10 +86,12 @@ class CheckedStream { return value; } std::size_t Tell() const { return source_.Tell(); } - Ch* PutBegin() { throw std::logic_error("In-situ parsing is disabled"); } - void Put(Ch) { throw std::logic_error("In-situ parsing is disabled"); } - void Flush() {} - std::size_t PutEnd(Ch*) { throw std::logic_error("In-situ parsing is disabled"); } + Ch* PutBegin() const { throw RapidJsonInputError("In-situ parsing is disabled"); } + void Put(Ch) const { throw RapidJsonInputError("In-situ parsing is disabled"); } + void Flush() const { + // Required by the RapidJSON stream concept; input has no output buffer. + } + std::size_t PutEnd(Ch*) const { throw RapidJsonInputError("In-situ parsing is disabled"); } }; struct FileCloser { @@ -88,7 +100,9 @@ struct FileCloser { struct DiscardStream { using Ch = char; - void Put(char) {} + void Put(char) const { + // Validate decoded UTF-8 without storing a second copy of the token. + } }; // kParseValidateEncodingFlag validates raw UTF-8, but RapidJSON 1.1 also @@ -144,7 +158,7 @@ class RJSAXHandler { std::fputc('\n', stdout) != EOF; return ok || fail("Unable to write JSON argument output"); } - bool integer(const std::string& value) { return argument(value.data(), value.size()); } + bool integer(std::string_view value) { return argument(value.data(), value.size()); } public: explicit RJSAXHandler(const rj_parser& settings) : settings_(settings) {} @@ -179,7 +193,8 @@ class RJSAXHandler { }; template -int parse(rj_parser* parser, Source& source, char** error_msg) { +int parse(const rj_parser* parser, Source& source, char** error_msg, + std::FILE* file = nullptr) { RJSAXHandler handler(*parser); // Fresh counters/path, retained settings. CheckedStream input(source); rapidjson::GenericReader, rapidjson::UTF8<>, @@ -187,6 +202,10 @@ int parse(rj_parser* parser, Source& source, char** error_msg) { constexpr unsigned flags = rapidjson::kParseNumbersAsStringsFlag | rapidjson::kParseValidateEncodingFlag | rapidjson::kParseIterativeFlag; const rapidjson::ParseResult result = reader.Parse(input, handler); + if (file != nullptr && std::ferror(file)) { + *error_msg = strdup("Error reading JSON file"); + return 2; + } if (!result) { std::string message = handler.error() ? handler.error() : rapidjson::GetParseError_En(result.Code()); @@ -201,7 +220,7 @@ int parse(rj_parser* parser, Source& source, char** error_msg) { return 0; } -int set_limit(rj_parser* parser, double value, unsigned int* target) { +int set_limit(const rj_parser* parser, double value, unsigned int* target) { if (parser == nullptr || !std::isfinite(value) || value < 0 || value > std::numeric_limits::max() || std::floor(value) != value) return -1; @@ -209,6 +228,25 @@ int set_limit(rj_parser* parser, double value, unsigned int* target) { return 0; } +// Contain exceptions at the C interface after handling the concrete failures +// produced by input validation, allocation, and standard string operations. +template +int at_c_boundary(Operation&& operation, char** error_msg, int failure_status = 2) noexcept { + try { + return operation(); + } catch (const RapidJsonInputError& error) { + *error_msg = strdup(error.what()); + } catch (const std::bad_alloc& error) { + *error_msg = strdup(error.what()); + } catch (const std::length_error& error) { + *error_msg = strdup(error.what()); + } catch (...) { + // Third-party exceptions of unknown type must never cross the C ABI. + *error_msg = strdup("Unexpected RapidJSON parser failure"); + } + return failure_status; +} + } // namespace extern "C" int rj_json_init(rj_parser** parser, char** error_msg) { @@ -216,16 +254,10 @@ extern "C" int rj_json_init(rj_parser** parser, char** error_msg) { *error_msg = nullptr; if (parser == nullptr) return -1; *parser = nullptr; - try { - *parser = new rj_parser{}; + return at_c_boundary([&] { + *parser = std::make_unique().release(); return 0; - } catch (const std::exception& e) { - *error_msg = strdup(e.what()); - return 1; - } catch (...) { - *error_msg = strdup("Unable to initialize RapidJSON parser"); - return 1; - } + }, error_msg, 1); } extern "C" int rj_parse_buffer(rj_parser* parser, const char* buf, @@ -233,16 +265,10 @@ extern "C" int rj_parse_buffer(rj_parser* parser, const char* buf, if (error_msg == nullptr) return -1; *error_msg = nullptr; if (parser == nullptr || buf == nullptr) return -1; - try { + return at_c_boundary([&] { rapidjson::MemoryStream input(buf, len); return parse(parser, input, error_msg); - } catch (const std::exception& e) { - *error_msg = strdup(e.what()); - return 2; - } catch (...) { - *error_msg = strdup("Unexpected RapidJSON parser failure"); - return 2; - } + }, error_msg); } extern "C" int rj_parse_file(rj_parser* parser, const char* filename, @@ -250,28 +276,16 @@ extern "C" int rj_parse_file(rj_parser* parser, const char* filename, if (error_msg == nullptr) return -1; *error_msg = nullptr; if (parser == nullptr || filename == nullptr) return -1; - try { - std::unique_ptr file(std::fopen(filename, "rb")); + return at_c_boundary([&] { + const std::unique_ptr file(std::fopen(filename, "rb")); if (!file) { *error_msg = strdup("Unable to open JSON file"); return 2; } - char buffer[16384]; - rapidjson::FileReadStream input(file.get(), buffer, sizeof(buffer)); - const int result = parse(parser, input, error_msg); - if (std::ferror(file.get())) { - std::free(*error_msg); - *error_msg = strdup("Error reading JSON file"); - return 2; - } - return result; - } catch (const std::exception& e) { - *error_msg = strdup(e.what()); - return 2; - } catch (...) { - *error_msg = strdup("Unexpected RapidJSON parser failure"); - return 2; - } + std::array buffer{}; + rapidjson::FileReadStream input(file.get(), buffer.data(), buffer.size()); + return parse(parser, input, error_msg, file.get()); + }, error_msg); } extern "C" int rj_set_max_depth(rj_parser* parser, double value) { @@ -286,7 +300,7 @@ extern "C" int rj_set_silence(rj_parser* parser, int silence) { return 0; } extern "C" int rj_json_cleanup(rj_parser* parser) { - delete parser; + const std::unique_ptr owned(parser); return 0; } diff --git a/tests/sax_adapter_api_tests.cpp b/tests/sax_adapter_api_tests.cpp index 8363325..94d16b2 100644 --- a/tests/sax_adapter_api_tests.cpp +++ b/tests/sax_adapter_api_tests.cpp @@ -2,107 +2,28 @@ // public C APIs for zero limits, rejected setters, reuse, and binary-safe I/O. #include "../src/rjparser.h" #include "../src/nlparser.h" +#include "test_support.h" -#include #include #include -#include +#include #include -#include -#include #include +#include #include #include -#include - #if HAVE_RAPIDJSON || HAVE_NLOHMANNJSON namespace { +using namespace std::string_view_literals; -class TemporaryInput { - std::string path_; - public: - explicit TemporaryInput(const std::string& text) { - char name[] = "/tmp/jsonbench-sax-api-XXXXXX"; - const int fd = mkstemp(name); - if (fd < 0) throw std::runtime_error("mkstemp failed"); - path_ = name; - std::size_t offset = 0; - while (offset < text.size()) { - const ssize_t count = write(fd, text.data() + offset, text.size() - offset); - if (count < 0 && errno == EINTR) continue; - if (count <= 0) { - close(fd); - unlink(path_.c_str()); - throw std::runtime_error("temporary input write failed"); - } - offset += static_cast(count); - } - if (close(fd) != 0) { - unlink(path_.c_str()); - throw std::runtime_error("temporary input close failed"); - } - } - ~TemporaryInput() { unlink(path_.c_str()); } - TemporaryInput(const TemporaryInput&) = delete; - TemporaryInput& operator=(const TemporaryInput&) = delete; - const char* path() const { return path_.c_str(); } -}; - -class CapturedOutput { - FILE* file_ = nullptr; - int saved_fd_ = -1; - public: - CapturedOutput() { - if (std::fflush(stdout) != 0) throw std::runtime_error("stdout flush failed"); - file_ = std::tmpfile(); - if (!file_) throw std::runtime_error("stdout temporary file failed"); - saved_fd_ = dup(STDOUT_FILENO); - if (saved_fd_ < 0 || dup2(fileno(file_), STDOUT_FILENO) < 0) { - if (saved_fd_ >= 0) close(saved_fd_); - std::fclose(file_); - throw std::runtime_error("stdout capture failed"); - } - } - ~CapturedOutput() { - std::fflush(stdout); - dup2(saved_fd_, STDOUT_FILENO); - close(saved_fd_); - std::fclose(file_); - } - CapturedOutput(const CapturedOutput&) = delete; - CapturedOutput& operator=(const CapturedOutput&) = delete; - std::string read() { - if (std::fflush(stdout) != 0 || std::fseek(file_, 0, SEEK_SET) != 0) - throw std::runtime_error("captured output rewind failed"); - std::string result; - char buffer[1024]; - std::size_t length; - while ((length = std::fread(buffer, 1, sizeof buffer, file_)) != 0) - result.append(buffer, length); - if (std::ferror(file_)) throw std::runtime_error("captured output read failed"); - return result; - } -}; - -// The API replaces the slot and returns free()-owned diagnostics. It must not -// free the caller's previous allocation, including on invalid-argument paths. -class ErrorSlot { - char* original_; - public: - char* value; - ErrorSlot() : original_(static_cast(std::malloc(1))), value(original_) { - if (!original_) throw std::bad_alloc(); - *original_ = '\0'; - } - ~ErrorSlot() { - if (value != original_) std::free(value); - std::free(original_); - } - ErrorSlot(const ErrorSlot&) = delete; - ErrorSlot& operator=(const ErrorSlot&) = delete; - bool replaced() const { return value != original_; } -}; +using jsonbench::test::CapturedOutput; +using jsonbench::test::ErrorSlot; +using jsonbench::test::TemporaryInput; +using jsonbench::test::TestError; +using jsonbench::test::parse_path; +template +using Parser = jsonbench::test::ApiParser; #if HAVE_RAPIDJSON struct RapidJsonApi { @@ -132,87 +53,37 @@ struct NlohmannApi { }; #endif -template -class Parser { - public: - typename Api::Handle* value = nullptr; - Parser() { - ErrorSlot error; - if (Api::init(&value, &error.value) != 0 || !value || error.value) { - Api::cleanup(value); - throw std::runtime_error(std::string(Api::name()) + " initialization failed"); - } - } - ~Parser() { Api::cleanup(value); } - Parser(const Parser&) = delete; - Parser& operator=(const Parser&) = delete; -}; - -struct Result { - int status; - bool replaced; - bool null_error; - std::string error; - std::string output; -}; +using Result = jsonbench::test::ParseResult; template -Result parse_path(typename Api::Handle* parser, const char* path) { - ErrorSlot error; - CapturedOutput output; - const int status = Api::file(parser, path, &error.value); - return {status, error.replaced(), !error.value, error.value ? error.value : "", output.read()}; -} - -template -Result parse(typename Api::Handle* parser, const std::string& text, bool file = false) { +Result parse(typename Api::Handle* parser, std::string_view text, bool file = false) { if (file) { TemporaryInput input(text); return parse_path(parser, input.path()); } ErrorSlot error; CapturedOutput output; - const int status = Api::buffer(parser, text.data(), static_cast(text.size()), &error.value); - return {status, error.replaced(), !error.value, error.value ? error.value : "", output.read()}; + const int status = Api::buffer(parser, text.data(), static_cast(text.size()), error.slot()); + return {status, error.replaced(), !error.get(), error.get() ? error.get() : "", output.read()}; } -class Checks { - unsigned int total_ = 0; - unsigned int failures_ = 0; +class Checks : public jsonbench::test::TestChecks { public: - std::string engine; - void check(const std::string& name, bool passed) { - ++total_; - if (!passed) { - ++failures_; - std::fprintf(stderr, "FAIL: %s: %s\n", engine.c_str(), name.c_str()); - } - } - void success(const std::string& name, const Result& result, const std::string& output = "") { - check(name + ": succeeds", result.status == 0); - check(name + ": clears old diagnostic", result.replaced && result.null_error); - check(name + ": byte-exact output", result.output == output); - if (result.status != 0) - std::fprintf(stderr, " diagnostic: %s\n", result.error.c_str()); - } - void failure(const std::string& name, const Result& result) { - check(name + ": fails", result.status != 0); - check(name + ": fresh malloc-owned diagnostic", result.replaced && !result.null_error && !result.error.empty()); + void success(std::string_view name, const Result& result, std::string_view expected = "") { + success3(name, result, expected, {": succeeds", ": clears old diagnostic", ": byte-exact output"}); } - unsigned int total() const { return total_; } - unsigned int failures() const { return failures_; } - int finish() const { - std::printf("SAX adapter C APIs: %u checks passed, %u failed\n", total_ - failures_, failures_); - return failures_ ? 1 : 0; + void failure(std::string_view name, const Result& result) { + failure2(name, result, result.status != 0, {": fails", ": fresh malloc-owned diagnostic"}); } + int finish() const { return report("SAX adapter C APIs"); } }; template void invalid_arguments(Checks& checks) { { ErrorSlot error; - checks.check("init rejects null parser slot", Api::init(nullptr, &error.value) != 0); - checks.check("invalid init clears old diagnostic", error.replaced() && !error.value); + checks.check("init rejects null parser slot", Api::init(nullptr, error.slot()) != 0); + checks.check("invalid init clears old diagnostic", error.replaced() && !error.get()); } typename Api::Handle* handle = nullptr; checks.check("init rejects null diagnostic slot", Api::init(&handle, nullptr) != 0); @@ -226,19 +97,19 @@ void invalid_arguments(Checks& checks) { TemporaryInput input("null"); for (unsigned int kind = 0; kind < 3; ++kind) { ErrorSlot error; - const int rc = kind == 0 ? Api::buffer(nullptr, "{}", 2, &error.value) : - Api::buffer(parser.value, nullptr, kind == 1 ? 0 : 1, &error.value); + const int rc = kind == 0 ? Api::buffer(nullptr, "{}", 2, error.slot()) : + Api::buffer(parser.value, nullptr, kind == 1 ? 0 : 1, error.slot()); checks.check("invalid buffer arguments " + std::to_string(kind), rc != 0); checks.check("invalid buffer replaces diagnostic " + std::to_string(kind), error.replaced()); checks.check("invalid buffer diagnostic is null or nonempty " + std::to_string(kind), - !error.value || *error.value); + !error.get() || *error.get()); } - for (const auto& invalid : std::vector>{ + for (const auto& [name, result] : std::vector>{ {"null parser", parse_path(nullptr, input.path())}, {"null filename", parse_path(parser.value, nullptr)}}) { - checks.check("file rejects " + invalid.first, invalid.second.status != 0); - checks.check("file replaces diagnostic for " + invalid.first, invalid.second.replaced); - checks.check("invalid file emits no output: " + invalid.first, invalid.second.output.empty()); + checks.check("file rejects " + name, result.status != 0); + checks.check("file replaces diagnostic for " + name, result.replaced); + checks.check("invalid file emits no output: " + name, result.output.empty()); } checks.check("buffer rejects null diagnostic slot", Api::buffer(parser.value, "{}", 2, nullptr) != 0); checks.check("file rejects null diagnostic slot", Api::file(parser.value, input.path(), nullptr) != 0); @@ -253,7 +124,7 @@ void limits(Checks& checks) { checks.check("set depth two", Api::depth(parser.value, 2) == 0); checks.check("set arguments two", Api::arguments(parser.value, 2) == 0); checks.check("enable silence", Api::silence(parser.value, 1) == 0); - const double upper = static_cast(UINT_MAX); + const auto upper = static_cast(UINT_MAX); const std::vector> invalid = { {"NaN", std::numeric_limits::quiet_NaN()}, {"infinity", std::numeric_limits::infinity()}, @@ -262,31 +133,31 @@ void limits(Checks& checks) { {"fraction below one", 0.5}, {"positive fraction", 1.5}, {"above UINT_MAX", upper + 1.0} }; - for (const auto& limit : invalid) { - checks.check("depth rejects " + limit.first, Api::depth(parser.value, limit.second) != 0); - checks.check("argument count rejects " + limit.first, Api::arguments(parser.value, limit.second) != 0); - checks.success("rejected " + limit.first + " preserves limits", parse(parser.value, "{\"a\":[null,true]}")); - checks.failure("rejected " + limit.first + " preserves depth ceiling", parse(parser.value, "{\"a\":[[null]]}")); - checks.failure("rejected " + limit.first + " preserves argument ceiling", parse(parser.value, "[null,true,false]")); + for (const auto& [name, value] : invalid) { + checks.check("depth rejects " + name, Api::depth(parser.value, value) != 0); + checks.check("argument count rejects " + name, Api::arguments(parser.value, value) != 0); + checks.success("rejected " + name + " preserves limits", parse(parser.value, R"({"a":[null,true]})")); + checks.failure("rejected " + name + " preserves depth ceiling", parse(parser.value, R"({"a":[[null]]})")); + checks.failure("rejected " + name + " preserves argument ceiling", parse(parser.value, "[null,true,false]")); } checks.check("depth accepts UINT_MAX", Api::depth(parser.value, upper) == 0); checks.check("argument count accepts UINT_MAX", Api::arguments(parser.value, upper) == 0); - checks.success("maximum limits permit ordinary JSON", parse(parser.value, "{\"a\":[null,true,false]}")); + checks.success("maximum limits permit ordinary JSON", parse(parser.value, R"({"a":[null,true,false]})")); checks.check("depth accepts zero", Api::depth(parser.value, 0) == 0); - for (const auto& value : {"null", "true", "false", "1", "\"value\""}) + for (const auto& value : {"null", "true", "false", "1", R"("value")"}) checks.success(std::string("zero depth allows root ") + value, parse(parser.value, value)); for (const auto& value : {"{}", "[]"}) checks.failure(std::string("zero depth rejects ") + value, parse(parser.value, value)); checks.check("restore depth three", Api::depth(parser.value, 3) == 0); checks.check("argument count accepts zero", Api::arguments(parser.value, 0) == 0); - for (const auto& value : {"{}", "[]", "{\"a\":[{},[]]}"}) + for (const auto& value : {"{}", "[]", R"({"a":[{},[]]})"}) checks.success(std::string("zero arguments allows ") + value, parse(parser.value, value)); - for (const auto& value : {"null", "true", "false", "0", "\"\"", "{\"a\":null}", "[false]"}) + for (const auto& value : {"null", "true", "false", "0", R"("")", R"({"a":null})", "[false]"}) checks.failure(std::string("zero arguments rejects ") + value, parse(parser.value, value)); checks.check("restore arguments two", Api::arguments(parser.value, 2) == 0); checks.success("exact depth", parse(parser.value, "[[[null]]]")); checks.failure("above depth", parse(parser.value, "[[[[null]]]]")); - checks.success("bool and null exactly at count", parse(parser.value, "{\"a\":null,\"b\":false}")); + checks.success("bool and null exactly at count", parse(parser.value, R"({"a":null,"b":false})")); checks.failure("bool and null above count", parse(parser.value, "[null,true,false]")); checks.check("negative zero is a valid zero depth", Api::depth(parser.value, -0.0) == 0); checks.failure("negative zero has zero-limit semantics", parse(parser.value, "{}")); @@ -299,33 +170,33 @@ void reuse_and_output(Checks& checks) { checks.check("set reuse arguments", Api::arguments(parser.value, 2) == 0); for (bool file : {false, true}) { const std::string kind = file ? "file" : "buffer"; - const Result truncated = parse(parser.value, "{\"leaked\":{\"x\":true,\"y\":", file); + const Result truncated = parse(parser.value, R"({"leaked":{"x":true,"y":)", file); checks.failure(kind + " truncated input", truncated); checks.check(kind + " emits completed prefix only", truncated.output == "leaked.x: true\n"); - checks.success(kind + " fresh path after failure", parse(parser.value, "{\"fresh\":false}", file), "fresh: false\n"); + checks.success(kind + " fresh path after failure", parse(parser.value, R"({"fresh":false})", file), "fresh: false\n"); checks.success(kind + " root array", parse(parser.value, "[null,true]", file), "array.array: \narray.array: true\n"); checks.success(kind + " root scalar resets array path", parse(parser.value, "2", file), ": 2\n"); const Result count = parse(parser.value, "[null,true,false]", file); checks.failure(kind + " retained argument limit", count); checks.check(kind + " only permitted arguments printed", count.output == "array.array: \narray.array: true\n"); - checks.success(kind + " resets count after failure", parse(parser.value, "{\"a\":true,\"b\":null}", file), "a: true\nb: \n"); + checks.success(kind + " resets count after failure", parse(parser.value, R"({"a":true,"b":null})", file), "a: true\nb: \n"); checks.failure(kind + " retained depth limit", parse(parser.value, "[[[null]]]", file)); - checks.success(kind + " resets after depth failure", parse(parser.value, "{\"after\":null}", file), "after: \n"); - const char binary[] = "a\0b.c\0d: v\0x\na\0b.after: false\n"; + checks.success(kind + " resets after depth failure", parse(parser.value, R"({"after":null})", file), "after: \n"); + constexpr auto binary = "a\0b.c\0d: v\0x\na\0b.after: false\n"sv; checks.success(kind + " embedded NUL values and paths", parse(parser.value, - "{\"a\\u0000b\":{\"c\\u0000d\":\"v\\u0000x\",\"after\":false}}", file), - std::string(binary, sizeof binary - 1)); + R"({"a\u0000b":{"c\u0000d":"v\u0000x","after":false}})", file), + std::string(binary)); checks.success(kind + " full dotted parent restoration", parse(parser.value, - "{\"a.b\":{\"c.d\":1},\"after\":2}", file), "a.b.c.d: 1\nafter: 2\n"); + R"({"a.b":{"c.d":1},"after":2})", file), "a.b.c.d: 1\nafter: 2\n"); checks.success(kind + " EOF root number is finalized", parse(parser.value, "1.250e+3", file), ": 1.250e+3\n"); checks.failure(kind + " empty input", parse(parser.value, "", file)); checks.failure(kind + " truncated EOF number", parse(parser.value, "1e+", file)); } checks.check("nonzero silence accepted", Api::silence(parser.value, -7) == 0); - checks.success("silent parsing validates and suppresses output", parse(parser.value, "{\"a\":null,\"b\":true}")); + checks.success("silent parsing validates and suppresses output", parse(parser.value, R"({"a":null,"b":true})")); checks.failure("silence retains argument limit", parse(parser.value, "[null,true,false]")); - checks.failure("silence retains syntax validation", parse(parser.value, "{\"a\":true,}")); - checks.success("silence persists across failed file reuse", parse(parser.value, "{\"quiet\":false}", true)); + checks.failure("silence retains syntax validation", parse(parser.value, R"({"a":true,})")); + checks.success("silence persists across failed file reuse", parse(parser.value, R"({"quiet":false})", true)); } template @@ -339,7 +210,7 @@ void stream_boundaries(Checks& checks) { std::string(" raw NUL at offset ") + std::to_string(offset); const Result result = parse(parser.value, raw_nul, file); checks.failure(label, result); - checks.check(label + ": no silent output", result.output.empty()); + checks.check(std::string(label) + ": no silent output", result.output.empty()); } } const std::string trailing = "{}" + std::string(16382, ' ') + "[]"; @@ -348,16 +219,16 @@ void stream_boundaries(Checks& checks) { checks.check("disable silence for valid boundary output", Api::silence(parser.value, 0) == 0); const std::string unicode = std::string(16382, ' ') + "\"\xc3\xa4\xf0\x9f\x98\x80\""; checks.success("UTF-8 spans file buffer boundary", parse(parser.value, unicode, true), ": \xc3\xa4\xf0\x9f\x98\x80\n"); - const std::string escape = std::string(16382, ' ') + "\"\\u0000x\""; - const char binary[] = ": \0x\n"; + const std::string escape = std::string(16382, ' ') + R"("\u0000x")"; + constexpr auto binary = ": \0x\n"sv; checks.success("decoded NUL escape spans file buffer boundary", parse(parser.value, escape, true), - std::string(binary, sizeof binary - 1)); - checks.success("reused adapter after boundary failures", parse(parser.value, "{\"fresh\":true}"), "fresh: true\n"); + std::string(binary)); + checks.success("reused adapter after boundary failures", parse(parser.value, R"({"fresh":true})"), "fresh: true\n"); } template void run(Checks& checks) { - checks.engine = Api::name(); + checks.set_engine(Api::name()); const unsigned int initial_total = checks.total(); const unsigned int initial_failures = checks.failures(); invalid_arguments(checks); From da812b3528ffc6b1324e9412d42309a7d9f5693e Mon Sep 17 00:00:00 2001 From: jens Steinbacher Date: Tue, 13 Oct 2026 17:10:37 +0000 Subject: [PATCH 06/18] Split native parser validation into focused checked helpers --- src/cjsonparser.c | 21 +- src/domarguments.h | 768 ++++++++++++++++++++++---------- src/janssonparser.c | 15 +- src/jsoncparser.c | 119 +++-- src/jsoncppparser.cpp | 230 ++++++---- src/simdjsonparser.cpp | 14 +- src/yyjsonparser.c | 15 +- tests/Makefile.am | 1 + tests/cpp_adapter_api_tests.cpp | 292 ++++-------- tests/dom_adapter_api_tests.c | 26 +- 10 files changed, 903 insertions(+), 598 deletions(-) diff --git a/src/cjsonparser.c b/src/cjsonparser.c index 754b08f..304d3f1 100644 --- a/src/cjsonparser.c +++ b/src/cjsonparser.c @@ -5,14 +5,16 @@ struct cjson_parser { dom_settings settings; }; -static int cjson_kind(const void *node) { - return cJSON_IsObject((const cJSON *)node) ? DOM_OBJECT : cJSON_IsArray((const cJSON *)node) ? DOM_ARRAY : DOM_SCALAR; +static int cjson_kind(void *node) { + if (cJSON_IsObject((cJSON *)node)) return DOM_OBJECT; + if (cJSON_IsArray((cJSON *)node)) return DOM_ARRAY; + return DOM_SCALAR; } -static int cjson_child(const void *node, dom_cursor *cursor, int first, - const void **child, const char **key, size_t *key_length) { - const cJSON *json = (const cJSON *)node; - const cJSON *item = first ? json->child : ((const cJSON *)cursor->pointer)->next; +static int cjson_child(void *node, dom_cursor *cursor, int first, + void **child, const char **key, size_t *key_length) { + cJSON *json = (cJSON *)node; + cJSON *item = first ? json->child : ((cJSON *)cursor->pointer)->next; if (item == NULL) return 0; cursor->pointer = (void *)item; *child = item; if (cJSON_IsObject(json)) { @@ -21,8 +23,8 @@ static int cjson_child(const void *node, dom_cursor *cursor, int first, return 1; } -static int cjson_scalar(const void *node, dom_value *value, char **error) { - const cJSON *json = (const cJSON *)node; +static int cjson_scalar(void *node, dom_value *value, char **error) { + cJSON *json = (cJSON *)node; if (cJSON_IsNull(json)) { value->data = ""; value->length = 0; } else if (cJSON_IsBool(json)) { value->data = cJSON_IsTrue(json) ? "true" : "false"; value->length = strlen(value->data); @@ -43,7 +45,8 @@ static const dom_adapter cjson_adapter = {cjson_kind, cjson_child, cjson_scalar} int cjson_parse_file(cjson_parser *parser, const char *filename, char **error_msg) { char *content = NULL; - size_t length = 0, keys = 0; + size_t length = 0; + size_t keys = 0; dom_numbers numbers = {0}; dom_locale_guard locale = {0}; const char *end = NULL; diff --git a/src/domarguments.h b/src/domarguments.h index aed6cd1..a232569 100644 --- a/src/domarguments.h +++ b/src/domarguments.h @@ -15,7 +15,8 @@ #include typedef struct { - unsigned int max_depth, max_args; + unsigned int max_depth; + unsigned int max_args; int silence; } dom_settings; @@ -31,7 +32,10 @@ static int dom_error(char **error, const char *message) { } #if HAVE_NEWLOCALE && HAVE_USELOCALE && HAVE_FREELOCALE && HAVE_DUPLOCALE && defined(LC_NUMERIC_MASK) -typedef struct { locale_t current, previous; } dom_locale_guard; +typedef struct { + locale_t current; + locale_t previous; +} dom_locale_guard; /* JSON numbers always use '.', regardless of the embedding application's * process or thread locale. Change only this thread and restore its previous @@ -91,7 +95,8 @@ static int dom_set_limit(double value, unsigned int *limit) { static int dom_read_file(const char *filename, char **content, size_t *length, char **error) { FILE *file = fopen(filename, "rb"); - size_t used = 0, capacity = 16384; + size_t used = 0; + size_t capacity = 16384; char *buffer; if (file == NULL) return dom_error(error, "Unable to open JSON file"); buffer = (char *)malloc(capacity); @@ -162,8 +167,10 @@ static int dom_code_unit(const char *data, size_t length, size_t *position, /* Strict scalar UTF-8, excluding overlong forms, surrogates and > U+10ffff. */ static int dom_utf8(const char *data, size_t length, size_t *position) { unsigned char first = (unsigned char)data[*position]; - size_t count, i; - unsigned int code, minimum; + size_t count; + size_t i; + unsigned int code; + unsigned int minimum; if (first < 0x80) { ++*position; return 1; } if (first >= 0xc2 && first <= 0xdf) { count = 2; code = first & 31; minimum = 0x80; } else if (first >= 0xe0 && first <= 0xef) { count = 3; code = first & 15; minimum = 0x800; } @@ -192,7 +199,8 @@ typedef struct { typedef struct { dom_number_span *spans; - size_t count, capacity; + size_t count; + size_t capacity; } dom_numbers; static int dom_remember_number(dom_numbers *numbers, const char *data, @@ -209,131 +217,308 @@ static int dom_remember_number(dom_numbers *numbers, const char *data, } numbers->spans[numbers->count].data = data; numbers->spans[numbers->count].length = length; - numbers->spans[numbers->count++].kind = kind; + numbers->spans[numbers->count].kind = kind; + ++numbers->count; return 0; } /* Tokens are checked before native parsing so hostile long strings/numbers * and deep structures cannot first allocate an unbounded native token/stack. * No structure acceptance depends on this scan: native parsing follows. */ +typedef struct { + const char *data; + size_t length; + size_t position; + size_t depth; + size_t arguments; + const dom_settings *settings; + int nul_policy; + int number_policy; + size_t *keys; + dom_numbers *numbers; + char **error; +} dom_scan; + +static int dom_scan_argument(dom_scan *scan) { + ++scan->arguments; + if (scan->arguments > scan->settings->max_args) + return dom_error(scan->error, "Argument number limit exceeded"); + return 0; +} + +static size_t dom_codepoint_length(unsigned int code) { + if (code < 0x80) return 1; + if (code < 0x800) return 2; + return 3; +} + +static int dom_scan_surrogate_pair(dom_scan *scan, size_t *decoded) { + unsigned int low; + if (scan->length - scan->position < 2 || scan->data[scan->position] != '\\' || + scan->data[scan->position + 1] != 'u') + return dom_error(scan->error, "Unpaired Unicode surrogate"); + scan->position += 2; + if (!dom_code_unit(scan->data, scan->length, &scan->position, &low) || + low < 0xdc00 || low > 0xdfff) + return dom_error(scan->error, "Invalid Unicode surrogate pair"); + *decoded += 4; + return 0; +} + +static int dom_scan_unicode_escape(dom_scan *scan, size_t *decoded, int *contains_nul) { + unsigned int code; + if (!dom_code_unit(scan->data, scan->length, &scan->position, &code)) + return dom_error(scan->error, "Invalid Unicode escape"); + if (code >= 0xd800 && code <= 0xdbff) return dom_scan_surrogate_pair(scan, decoded); + if (code >= 0xdc00 && code <= 0xdfff) + return dom_error(scan->error, "Unpaired Unicode surrogate"); + if (code == 0) *contains_nul = 1; + *decoded += dom_codepoint_length(code); + return 0; +} + +static int dom_scan_escape(dom_scan *scan, size_t *decoded, int *contains_nul) { + unsigned char c; + if (scan->position == scan->length) return dom_error(scan->error, "Truncated JSON escape"); + c = (unsigned char)scan->data[scan->position]; + ++scan->position; + if (c == 'u') return dom_scan_unicode_escape(scan, decoded, contains_nul); + switch (c) { + case '"': case '\\': case '/': case 'b': case 'f': case 'n': case 'r': case 't': + ++*decoded; + return 0; + default: + return dom_error(scan->error, "Invalid JSON escape"); + } +} + +static int dom_scan_string_character(dom_scan *scan, unsigned char c, + size_t *decoded, int *contains_nul) { + size_t start; + if (c < 0x20) return dom_error(scan->error, "Invalid control character in JSON string"); + if (c == '\\') return dom_scan_escape(scan, decoded, contains_nul); + if (c < 0x80) { + ++*decoded; + return 0; + } + --scan->position; + start = scan->position; + if (!dom_utf8(scan->data, scan->length, &scan->position)) + return dom_error(scan->error, "Invalid UTF-8 in JSON string"); + *decoded += scan->position - start; + return 0; +} + +static int dom_scan_string_role(dom_scan *scan, int contains_nul) { + size_t end = scan->position; + while (end < scan->length && dom_space((unsigned char)scan->data[end])) ++end; + if (end < scan->length && scan->data[end] == ':') { + ++*scan->keys; + if (contains_nul && scan->nul_policy != DOM_NUL_ALLOWED) + return dom_error(scan->error, "Native DOM cannot preserve NUL in object keys"); + return 0; + } + if (contains_nul && scan->nul_policy == DOM_NUL_FORBIDDEN) + return dom_error(scan->error, "Native DOM cannot preserve NUL in string values"); + return dom_scan_argument(scan); +} + +static int dom_scan_string(dom_scan *scan) { + size_t decoded = 0; + int contains_nul = 0; + int closed = 0; + ++scan->position; + while (scan->position < scan->length) { + unsigned char c = (unsigned char)scan->data[scan->position]; + int result; + ++scan->position; + if (c == '"') { + closed = 1; + break; + } + result = dom_scan_string_character(scan, c, &decoded, &contains_nul); + if (result != 0) return result; + if (decoded >= JSON_STRING_SIZE) + return dom_error(scan->error, "Argument name or value too long"); + } + if (!closed) return dom_error(scan->error, "Truncated JSON string"); + return dom_scan_string_role(scan, contains_nul); +} + +static int dom_digit(unsigned char c) { + return c >= '0' && c <= '9'; +} + +static void dom_scan_digits(dom_scan *scan) { + while (scan->position < scan->length && dom_digit((unsigned char)scan->data[scan->position])) + ++scan->position; +} + +static int dom_scan_integer_part(dom_scan *scan) { + if (scan->position == scan->length || !dom_digit((unsigned char)scan->data[scan->position])) + return dom_error(scan->error, "Invalid JSON number"); + if (scan->data[scan->position] == '0') ++scan->position; + else dom_scan_digits(scan); + return 0; +} + +static int dom_scan_fraction(dom_scan *scan, int *real) { + size_t start; + if (scan->position == scan->length || scan->data[scan->position] != '.') return 0; + *real = 1; + ++scan->position; + start = scan->position; + dom_scan_digits(scan); + if (scan->position == start) return dom_error(scan->error, "Invalid JSON fraction"); + return 0; +} + +static int dom_scan_exponent(dom_scan *scan, int *real) { + size_t start; + unsigned char c; + if (scan->position == scan->length) return 0; + c = (unsigned char)scan->data[scan->position]; + if (c != 'e' && c != 'E') return 0; + *real = 1; + ++scan->position; + if (scan->position < scan->length && + (scan->data[scan->position] == '+' || scan->data[scan->position] == '-')) ++scan->position; + start = scan->position; + dom_scan_digits(scan); + if (scan->position == start) return dom_error(scan->error, "Invalid JSON exponent"); + return 0; +} + +static const char *dom_integer_maximum(int policy, int negative) { + if (policy == DOM_NUM_DOUBLE) return "9007199254740992"; + if (negative) return "9223372036854775808"; + if (policy == DOM_NUM_SIGNED) return "9223372036854775807"; + return "18446744073709551615"; +} + +static int dom_scan_integer_range(dom_scan *scan, size_t digits, int negative) { + const char *maximum = dom_integer_maximum(scan->number_policy, negative); + size_t integer_length = scan->position - digits; + size_t maximum_length = strlen(maximum); + if (integer_length > maximum_length || + (integer_length == maximum_length && memcmp(scan->data + digits, maximum, maximum_length) > 0)) + return dom_error(scan->error, "JSON integer is outside the native DOM's lossless range"); + return 0; +} + +static int dom_scan_real_range(dom_scan *scan, size_t start, size_t length) { + char number[JSON_STRING_SIZE]; + char *end; + double value; + memcpy(number, scan->data + start, length); + number[length] = '\0'; + errno = 0; + value = strtod(number, &end); + if (*end != '\0' || !isfinite(value) || (errno == ERANGE && value == 0.0)) + return dom_error(scan->error, "JSON real is outside the native DOM's finite range"); + return 0; +} + +static int dom_scan_number_separator(const dom_scan *scan) { + unsigned char c; + if (scan->position == scan->length) return 1; + c = (unsigned char)scan->data[scan->position]; + return dom_space(c) || c == ',' || c == ']' || c == '}'; +} + +static int dom_scan_number_range(dom_scan *scan, size_t start, size_t digits, + size_t length, int real, int negative) { + if (scan->number_policy == DOM_NUM_RAW) return 0; + if (real) return dom_scan_real_range(scan, start, length); + return dom_scan_integer_range(scan, digits, negative); +} + +static int dom_scan_number(dom_scan *scan) { + size_t start = scan->position; + size_t digits; + size_t length; + int negative = scan->data[start] == '-'; + int real = 0; + int result; + if (negative) ++scan->position; + digits = scan->position; + result = dom_scan_integer_part(scan); + if (result != 0) return result; + result = dom_scan_fraction(scan, &real); + if (result != 0) return result; + result = dom_scan_exponent(scan, &real); + if (result != 0) return result; + if (!dom_scan_number_separator(scan)) return dom_error(scan->error, "Invalid JSON number separator"); + length = scan->position - start; + if (length >= JSON_STRING_SIZE) return dom_error(scan->error, "Argument value too long"); + result = dom_scan_number_range(scan, start, digits, length, real, negative); + if (result != 0) return result; + result = dom_scan_argument(scan); + if (result != 0) return result; + return dom_remember_number(scan->numbers, scan->data + start, length, + real ? DOM_REAL : DOM_INTEGER, scan->error); +} + +static const char *dom_literal(unsigned char c) { + if (c == 't') return "true"; + if (c == 'f') return "false"; + return "null"; +} + +static int dom_scan_literal(dom_scan *scan, unsigned char c) { + const char *literal = dom_literal(c); + size_t length = strlen(literal); + if (scan->length - scan->position < length || + memcmp(scan->data + scan->position, literal, length) != 0) + return dom_error(scan->error, "Invalid JSON literal"); + scan->position += length; + return dom_scan_argument(scan); +} + +static int dom_scan_token(dom_scan *scan) { + unsigned char c = (unsigned char)scan->data[scan->position]; + if (dom_space(c) || c == ':' || c == ',') { + ++scan->position; + return 0; + } + switch (c) { + case '{': case '[': + ++scan->depth; + if (scan->depth > scan->settings->max_depth) + return dom_error(scan->error, "JSON depth limit exceeded"); + ++scan->position; + return 0; + case '}': case ']': + if (scan->depth > 0) --scan->depth; + ++scan->position; + return 0; + case '"': + return dom_scan_string(scan); + case 't': case 'f': case 'n': + return dom_scan_literal(scan, c); + default: + if (c == '-' || dom_digit(c)) return dom_scan_number(scan); + return dom_error(scan->error, "Unexpected character in JSON input"); + } +} + static int dom_preflight_impl(const char *data, size_t length, const dom_settings *settings, int nul_policy, int number_policy, size_t *keys, dom_numbers *numbers, char **error) { - size_t position = 0, depth = 0, arguments = 0; + dom_scan scan; + memset(&scan, 0, sizeof(scan)); + scan.data = data; + scan.length = length; + scan.settings = settings; + scan.nul_policy = nul_policy; + scan.number_policy = number_policy; + scan.keys = keys; + scan.numbers = numbers; + scan.error = error; *keys = 0; - while (position < length) { - unsigned char c = (unsigned char)data[position]; - if (dom_space(c) || c == ':' || c == ',') { ++position; continue; } - if (c == '{' || c == '[') { - if (++depth > settings->max_depth) return dom_error(error, "JSON depth limit exceeded"); - ++position; continue; - } - if (c == '}' || c == ']') { if (depth > 0) --depth; ++position; continue; } - if (c == '"') { - size_t decoded = 0, end; - int closed = 0, contains_nul = 0; - ++position; - while (position < length) { - c = (unsigned char)data[position++]; - if (c == '"') { closed = 1; break; } - if (c < 0x20) return dom_error(error, "Invalid control character in JSON string"); - if (c == '\\') { - unsigned int code; - if (position == length) return dom_error(error, "Truncated JSON escape"); - c = (unsigned char)data[position++]; - if (c == 'u') { - if (!dom_code_unit(data, length, &position, &code)) return dom_error(error, "Invalid Unicode escape"); - if (code >= 0xd800 && code <= 0xdbff) { - unsigned int low; - if (length - position < 2 || data[position] != '\\' || data[position + 1] != 'u') - return dom_error(error, "Unpaired Unicode surrogate"); - position += 2; - if (!dom_code_unit(data, length, &position, &low) || low < 0xdc00 || low > 0xdfff) - return dom_error(error, "Invalid Unicode surrogate pair"); - decoded += 4; - } else { - if (code >= 0xdc00 && code <= 0xdfff) return dom_error(error, "Unpaired Unicode surrogate"); - if (code == 0) contains_nul = 1; - decoded += code < 0x80 ? 1 : code < 0x800 ? 2 : 3; - } - } else if (c == '"' || c == '\\' || c == '/' || c == 'b' || c == 'f' || c == 'n' || c == 'r' || c == 't') { - ++decoded; - } else return dom_error(error, "Invalid JSON escape"); - } else if (c >= 0x80) { - size_t start = --position; - if (!dom_utf8(data, length, &position)) return dom_error(error, "Invalid UTF-8 in JSON string"); - decoded += position - start; - } else ++decoded; - if (decoded >= JSON_STRING_SIZE) return dom_error(error, "Argument name or value too long"); - } - if (!closed) return dom_error(error, "Truncated JSON string"); - end = position; - while (end < length && dom_space((unsigned char)data[end])) ++end; - if (end < length && data[end] == ':') { - ++*keys; - if (contains_nul && nul_policy != DOM_NUL_ALLOWED) return dom_error(error, "Native DOM cannot preserve NUL in object keys"); - } else { - if (contains_nul && nul_policy == DOM_NUL_FORBIDDEN) return dom_error(error, "Native DOM cannot preserve NUL in string values"); - if (++arguments > settings->max_args) return dom_error(error, "Argument number limit exceeded"); - } - continue; - } - if (c == '-' || (c >= '0' && c <= '9')) { - size_t start = position, digits, token_length; - int negative = c == '-', real = 0; - if (negative) ++position; - digits = position; - if (position == length || data[position] < '0' || data[position] > '9') return dom_error(error, "Invalid JSON number"); - if (data[position] == '0') ++position; - else while (position < length && data[position] >= '0' && data[position] <= '9') ++position; - if (position < length && data[position] == '.') { - size_t fraction = ++position; - real = 1; - while (position < length && data[position] >= '0' && data[position] <= '9') ++position; - if (position == fraction) return dom_error(error, "Invalid JSON fraction"); - } - if (position < length && (data[position] == 'e' || data[position] == 'E')) { - size_t exponent; - real = 1; ++position; - if (position < length && (data[position] == '+' || data[position] == '-')) ++position; - exponent = position; - while (position < length && data[position] >= '0' && data[position] <= '9') ++position; - if (position == exponent) return dom_error(error, "Invalid JSON exponent"); - } - if (position < length && !dom_space((unsigned char)data[position]) && data[position] != ',' && data[position] != ']' && data[position] != '}') - return dom_error(error, "Invalid JSON number separator"); - token_length = position - start; - if (token_length >= JSON_STRING_SIZE) return dom_error(error, "Argument value too long"); - if (!real && number_policy != DOM_NUM_RAW) { - const char *maximum = number_policy == DOM_NUM_DOUBLE ? "9007199254740992" : - negative ? "9223372036854775808" : - number_policy == DOM_NUM_SIGNED ? "9223372036854775807" : "18446744073709551615"; - size_t integer_length = position - digits, max_length = strlen(maximum); - if (integer_length > max_length || (integer_length == max_length && memcmp(data + digits, maximum, max_length) > 0)) - return dom_error(error, "JSON integer is outside the native DOM's lossless range"); - } - if (real && number_policy != DOM_NUM_RAW) { - char number[JSON_STRING_SIZE], *end; - double value; - memcpy(number, data + start, token_length); number[token_length] = '\0'; - errno = 0; value = strtod(number, &end); - if (*end != '\0' || !isfinite(value) || (errno == ERANGE && value == 0.0)) - return dom_error(error, "JSON real is outside the native DOM's finite range"); - } - if (++arguments > settings->max_args) return dom_error(error, "Argument number limit exceeded"); - if (dom_remember_number(numbers, data + start, token_length, - real ? DOM_REAL : DOM_INTEGER, error) != 0) return 2; - continue; - } - if (c == 't' || c == 'f' || c == 'n') { - const char *literal = c == 't' ? "true" : c == 'f' ? "false" : "null"; - size_t count = strlen(literal); - if (length - position < count || memcmp(data + position, literal, count) != 0) return dom_error(error, "Invalid JSON literal"); - position += count; - if (++arguments > settings->max_args) return dom_error(error, "Argument number limit exceeded"); - continue; - } - return dom_error(error, "Unexpected character in JSON input"); + while (scan.position < scan.length) { + int result = dom_scan_token(&scan); + if (result != 0) return result; } return 0; } @@ -370,142 +555,249 @@ static inline int dom_preflight_numbers(const char *data, size_t length, enum { DOM_SCALAR = 0, DOM_OBJECT = 1, DOM_ARRAY = 2 }; /* Large/aligned enough for native cursor structs; adapters use memcpy. */ -typedef union { void *pointer; size_t index; void *slots[8]; long double alignment; } dom_cursor; -typedef struct { const char *data; size_t length; char number[128]; int number_kind; } dom_value; +typedef union { + void *pointer; + size_t index; + void *slots[8]; + long double alignment; +} dom_cursor; + +typedef struct { + const char *data; + size_t length; + char number[128]; + int number_kind; +} dom_value; + typedef struct { - int (*kind)(const void *node); - int (*child)(const void *node, dom_cursor *cursor, int first, - const void **child, const char **key, size_t *key_length); - int (*scalar)(const void *node, dom_value *value, char **error); + int (*kind)(void *node); + int (*child)(void *node, dom_cursor *cursor, int first, + void **child, const char **key, size_t *key_length); + int (*scalar)(void *node, dom_value *value, char **error); } dom_adapter; typedef struct { - const void *node; + void *node; dom_cursor cursor; int first; - size_t prefix_before, key_length; + size_t prefix_before; + size_t key_length; char *key_before; } dom_frame; -static int dom_walk_impl(const void *root, const dom_adapter *adapter, +typedef struct { + dom_frame *frames; + size_t depth; + size_t capacity; + size_t prefix_length; + size_t key_length; + unsigned int arguments; + size_t number_position; + char prefix[JSON_STRING_SIZE]; + char key[JSON_STRING_SIZE]; + void *node; + const dom_adapter *adapter; + const dom_settings *settings; + const dom_numbers *numbers; + size_t *object_keys; + char **error; +} dom_walk_state; + +static int dom_walk_grow_frames(dom_walk_state *state) { + size_t next; + dom_frame *grown; + if (state->depth < state->capacity) return 0; + next = state->capacity ? state->capacity * 2 : 16; + if (next < state->capacity || next > SIZE_MAX / sizeof(*state->frames)) + return dom_error(state->error, "DOM traversal stack overflow"); + grown = (dom_frame *)realloc(state->frames, next * sizeof(*state->frames)); + if (grown == NULL) return dom_error(state->error, "Unable to allocate DOM traversal stack"); + state->frames = grown; + state->capacity = next; + return 0; +} + +static size_t dom_walk_prefix_size(const dom_walk_state *state, int kind) { + if (kind == DOM_ARRAY && state->depth == 0) return 5; + return state->prefix_length + (state->prefix_length ? 1 : 0) + state->key_length; +} + +static void dom_walk_push_prefix(dom_walk_state *state, int kind) { + if (kind == DOM_ARRAY && state->depth == 0) { + memcpy(state->prefix, "array", 5); + memcpy(state->key, "array", 5); + state->prefix_length = 5; + state->key_length = 5; + return; + } + if (state->prefix_length) { + state->prefix[state->prefix_length] = '.'; + ++state->prefix_length; + } + memcpy(state->prefix + state->prefix_length, state->key, state->key_length); + state->prefix_length += state->key_length; +} + +static int dom_walk_begin(dom_walk_state *state, int kind) { + dom_frame *frame; + int result; + if (state->depth >= state->settings->max_depth) + return dom_error(state->error, "JSON depth limit exceeded"); + if (dom_walk_prefix_size(state, kind) >= JSON_STRING_SIZE) + return dom_error(state->error, "Argument name too long"); + result = dom_walk_grow_frames(state); + if (result != 0) return result; + frame = &state->frames[state->depth]; + memset(frame, 0, sizeof(*frame)); + frame->node = state->node; + frame->first = 1; + frame->prefix_before = state->prefix_length; + frame->key_length = state->key_length; + frame->key_before = (char *)malloc(state->key_length + 1); + if (frame->key_before == NULL) return dom_error(state->error, "Unable to allocate DOM traversal key"); + memcpy(frame->key_before, state->key, state->key_length); + dom_walk_push_prefix(state, kind); + ++state->depth; + return 0; +} + +static void dom_walk_pop(dom_walk_state *state) { + dom_frame *frame = &state->frames[state->depth - 1]; + state->prefix_length = frame->prefix_before; + state->key_length = frame->key_length; + memcpy(state->key, frame->key_before, state->key_length); + free(frame->key_before); + --state->depth; +} + +static int dom_walk_number_source(dom_walk_state *state, dom_value *value) { + const dom_number_span *span; + if (state->numbers == NULL || value->number_kind == DOM_NOT_NUMBER) return 0; + if (state->number_position >= state->numbers->count) + return dom_error(state->error, "Native DOM has unexpected numeric arguments"); + span = &state->numbers->spans[state->number_position]; + ++state->number_position; + if (value->number_kind != DOM_ANY_NUMBER && value->number_kind != span->kind) + return dom_error(state->error, "Native DOM numeric type differs from source token"); + value->data = span->data; + value->length = span->length; + return 0; +} + +static int dom_walk_write(const dom_walk_state *state, const dom_value *value) { + struct { + const char *data; + size_t length; + } parts[] = { + {state->prefix, state->prefix_length}, + {".", state->prefix_length ? 1U : 0U}, + {state->key, state->key_length}, + {": ", 2}, + {value->data, value->length}, + {"\n", 1} + }; + size_t i; + for (i = 0; i < sizeof(parts) / sizeof(parts[0]); ++i) { + if (parts[i].length == 0) continue; + if (fwrite(parts[i].data, 1, parts[i].length, stdout) != parts[i].length) + return dom_error(state->error, "Unable to write JSON argument output"); + } + return 0; +} + +static int dom_walk_argument(dom_walk_state *state) { + dom_value value; + size_t name_length = state->prefix_length + (state->prefix_length ? 1 : 0) + state->key_length; + int result; + if (state->arguments >= state->settings->max_args) + return dom_error(state->error, "Argument number limit exceeded"); + if (name_length >= JSON_STRING_SIZE) return dom_error(state->error, "Argument name too long"); + memset(&value, 0, sizeof(value)); + result = state->adapter->scalar(state->node, &value, state->error); + if (result != 0) return result; + result = dom_walk_number_source(state, &value); + if (result != 0) return result; + if (value.length >= JSON_STRING_SIZE) return dom_error(state->error, "Argument value too long"); + ++state->arguments; + if (state->settings->silence) return 0; + return dom_walk_write(state, &value); +} + +static int dom_walk_enter(dom_walk_state *state) { + int kind = state->adapter->kind(state->node); + if (kind == DOM_SCALAR) return dom_walk_argument(state); + return dom_walk_begin(state, kind); +} + +static int dom_walk_object_child(dom_walk_state *state, const char *key, + size_t length, int *entering) { + ++*state->object_keys; + if (length >= JSON_STRING_SIZE) return dom_error(state->error, "Argument name too long"); + memcpy(state->key, key, length); + state->key_length = length; + *entering = 1; + return 0; +} + +static int dom_walk_next(dom_walk_state *state, int *entering) { + *entering = 0; + while (state->depth) { + dom_frame *frame = &state->frames[state->depth - 1]; + const char *child_key = NULL; + size_t child_key_length = 0; + if (state->adapter->child(frame->node, &frame->cursor, frame->first, + &state->node, &child_key, &child_key_length)) { + frame->first = 0; + if (state->adapter->kind(frame->node) == DOM_OBJECT) + return dom_walk_object_child(state, child_key, child_key_length, entering); + *entering = 1; + return 0; + } + dom_walk_pop(state); + } + return 0; +} + +static int dom_walk_finish(dom_walk_state *state, int result) { + while (state->depth) dom_walk_pop(state); + free(state->frames); + if (result != 0) return result; + if (state->numbers != NULL && state->number_position != state->numbers->count) + return dom_error(state->error, "Native DOM omitted numeric arguments from source"); + if (!state->settings->silence && fflush(stdout) != 0) + return dom_error(state->error, "Unable to flush JSON argument output"); + return 0; +} + +static int dom_walk_impl(void *root, const dom_adapter *adapter, const dom_settings *settings, size_t *object_keys, const dom_numbers *numbers, char **error) { - dom_frame *frames = NULL; - size_t depth = 0, capacity = 0, prefix_length = 0, key_length = 0; - unsigned int arguments = 0; - size_t number_position = 0; - char prefix[JSON_STRING_SIZE], key[JSON_STRING_SIZE]; - const void *node = root; - int result = 0, entering = 1; + dom_walk_state state; + int entering = 1; + int result = 0; + memset(&state, 0, sizeof(state)); + state.node = root; + state.adapter = adapter; + state.settings = settings; + state.numbers = numbers; + state.object_keys = object_keys; + state.error = error; *object_keys = 0; - for (;;) { - if (entering) { - int kind = adapter->kind(node); - if (kind != DOM_SCALAR) { - dom_frame *frame; - size_t new_length = kind == DOM_ARRAY && depth == 0 ? 5 : - prefix_length + (prefix_length ? 1 : 0) + key_length; - if (depth >= settings->max_depth) { result = dom_error(error, "JSON depth limit exceeded"); break; } - if (new_length >= JSON_STRING_SIZE) { result = dom_error(error, "Argument name too long"); break; } - if (depth == capacity) { - dom_frame *grown; - size_t next = capacity ? capacity * 2 : 16; - if (next < capacity || next > SIZE_MAX / sizeof(*frames)) { result = dom_error(error, "DOM traversal stack overflow"); break; } - grown = (dom_frame *)realloc(frames, next * sizeof(*frames)); - if (grown == NULL) { result = dom_error(error, "Unable to allocate DOM traversal stack"); break; } - frames = grown; capacity = next; - } - frame = &frames[depth]; - memset(frame, 0, sizeof(*frame)); - frame->node = node; frame->first = 1; - frame->prefix_before = prefix_length; frame->key_length = key_length; - frame->key_before = (char *)malloc(key_length + 1); - if (frame->key_before == NULL) { result = dom_error(error, "Unable to allocate DOM traversal key"); break; } - memcpy(frame->key_before, key, key_length); - if (kind == DOM_ARRAY && depth == 0) { - memcpy(prefix, "array", 5); memcpy(key, "array", 5); - prefix_length = key_length = 5; - } else { - if (prefix_length) prefix[prefix_length++] = '.'; - memcpy(prefix + prefix_length, key, key_length); - prefix_length += key_length; - } - ++depth; - } else { - dom_value value; - size_t name_length = prefix_length + (prefix_length ? 1 : 0) + key_length; - memset(&value, 0, sizeof(value)); - if (arguments >= settings->max_args) { result = dom_error(error, "Argument number limit exceeded"); break; } - if (name_length >= JSON_STRING_SIZE) { result = dom_error(error, "Argument name too long"); break; } - result = adapter->scalar(node, &value, error); - if (result != 0) break; - if (numbers != NULL && value.number_kind != DOM_NOT_NUMBER) { - const dom_number_span *span; - if (number_position >= numbers->count) { - result = dom_error(error, "Native DOM has unexpected numeric arguments"); break; - } - span = &numbers->spans[number_position++]; - if (value.number_kind != DOM_ANY_NUMBER && value.number_kind != span->kind) { - result = dom_error(error, "Native DOM numeric type differs from source token"); break; - } - value.data = span->data; value.length = span->length; - } - if (value.length >= JSON_STRING_SIZE) { result = dom_error(error, "Argument value too long"); break; } - ++arguments; - if (!settings->silence) { - int ok = 1; - if (prefix_length) { - ok = fwrite(prefix, 1, prefix_length, stdout) == prefix_length; - ok = (fputc('.', stdout) != EOF) && ok; - } - ok = (fwrite(key, 1, key_length, stdout) == key_length) && ok; - ok = (fwrite(": ", 1, 2, stdout) == 2) && ok; - if (value.length) ok = (fwrite(value.data, 1, value.length, stdout) == value.length) && ok; - ok = (fputc('\n', stdout) != EOF) && ok; - if (!ok) { result = dom_error(error, "Unable to write JSON argument output"); break; } - } - } - } - entering = 0; - while (depth) { - dom_frame *frame = &frames[depth - 1]; - const char *child_key = NULL; - size_t child_key_length = 0; - if (adapter->child(frame->node, &frame->cursor, frame->first, - &node, &child_key, &child_key_length)) { - frame->first = 0; - if (adapter->kind(frame->node) == DOM_OBJECT) { - ++*object_keys; - if (child_key_length >= JSON_STRING_SIZE) { result = dom_error(error, "Argument name too long"); goto done; } - memcpy(key, child_key, child_key_length); key_length = child_key_length; - } - entering = 1; - break; - } - prefix_length = frame->prefix_before; - key_length = frame->key_length; - memcpy(key, frame->key_before, key_length); - free(frame->key_before); - --depth; - } - if (!entering) break; + while (entering) { + result = dom_walk_enter(&state); + if (result != 0) break; + result = dom_walk_next(&state, &entering); + if (result != 0) break; } -done: - while (depth) free(frames[--depth].key_before); - free(frames); - if (result == 0 && numbers != NULL && number_position != numbers->count) - result = dom_error(error, "Native DOM omitted numeric arguments from source"); - if (result == 0 && !settings->silence && fflush(stdout) != 0) - result = dom_error(error, "Unable to flush JSON argument output"); - return result; + return dom_walk_finish(&state, result); } -static inline int dom_walk(const void *root, const dom_adapter *adapter, +static inline int dom_walk(void *root, const dom_adapter *adapter, const dom_settings *settings, size_t *object_keys, char **error) { return dom_walk_impl(root, adapter, settings, object_keys, NULL, error); } -static inline int dom_walk_numbers(const void *root, const dom_adapter *adapter, +static inline int dom_walk_numbers(void *root, const dom_adapter *adapter, const dom_settings *settings, size_t *object_keys, const dom_numbers *numbers, char **error) { return dom_walk_impl(root, adapter, settings, object_keys, numbers, error); diff --git a/src/janssonparser.c b/src/janssonparser.c index e912a37..9bccc79 100644 --- a/src/janssonparser.c +++ b/src/janssonparser.c @@ -5,12 +5,14 @@ struct jansson_parser { dom_settings settings; }; -static int jansson_kind(const void *node) { - return json_is_object((json_t *)node) ? DOM_OBJECT : json_is_array((json_t *)node) ? DOM_ARRAY : DOM_SCALAR; +static int jansson_kind(void *node) { + if (json_is_object((json_t *)node)) return DOM_OBJECT; + if (json_is_array((json_t *)node)) return DOM_ARRAY; + return DOM_SCALAR; } -static int jansson_child(const void *node, dom_cursor *cursor, int first, - const void **child, const char **key, size_t *key_length) { +static int jansson_child(void *node, dom_cursor *cursor, int first, + void **child, const char **key, size_t *key_length) { json_t *json = (json_t *)node; if (json_is_array(json)) { if (first) cursor->index = 0; @@ -26,7 +28,7 @@ static int jansson_child(const void *node, dom_cursor *cursor, int first, return 1; } -static int jansson_scalar(const void *node, dom_value *value, char **error) { +static int jansson_scalar(void *node, dom_value *value, char **error) { json_t *json = (json_t *)node; int length; if (json_is_null(json)) { value->data = ""; value->length = 0; } @@ -54,7 +56,8 @@ static const dom_adapter jansson_adapter = {jansson_kind, jansson_child, jansson int jansson_parse_file(jansson_parser *parser, const char *filename, char **error_msg) { char *content = NULL; - size_t length = 0, keys = 0; + size_t length = 0; + size_t keys = 0; dom_numbers numbers = {0}; dom_locale_guard locale = {0}; json_error_t native_error; diff --git a/src/jsoncparser.c b/src/jsoncparser.c index 65d41bd..0c75eee 100644 --- a/src/jsoncparser.c +++ b/src/jsoncparser.c @@ -6,13 +6,15 @@ struct jsonc_parser { dom_settings settings; }; -static int jsonc_kind(const void *node) { +static int jsonc_kind(void *node) { enum json_type type = json_object_get_type((struct json_object *)node); - return type == json_type_object ? DOM_OBJECT : type == json_type_array ? DOM_ARRAY : DOM_SCALAR; + if (type == json_type_object) return DOM_OBJECT; + if (type == json_type_array) return DOM_ARRAY; + return DOM_SCALAR; } -static int jsonc_child(const void *node, dom_cursor *cursor, int first, - const void **child, const char **key, size_t *key_length) { +static int jsonc_child(void *node, dom_cursor *cursor, int first, + void **child, const char **key, size_t *key_length) { struct json_object *object = (struct json_object *)node; if (jsonc_kind(node) == DOM_ARRAY) { if (first) cursor->index = 0; @@ -29,7 +31,7 @@ static int jsonc_child(const void *node, dom_cursor *cursor, int first, return 1; } -static int jsonc_scalar(const void *node, dom_value *value, char **error) { +static int jsonc_scalar(void *node, dom_value *value, char **error) { struct json_object *object = (struct json_object *)node; switch (json_object_get_type(object)) { case json_type_null: value->data = ""; value->length = 0; break; @@ -68,49 +70,104 @@ static void jsonc_unicode_escape(char *output, size_t *used, unsigned int code) * Preserve strict native grammar checking by converting already verified * scalar UTF-8 to equivalent JSON Unicode escapes. This is a representation * conversion only; escaped characters and structural bytes are unchanged. */ -static int jsonc_ascii_input(const char *data, size_t length, char **ascii, - size_t *ascii_length, char **error) { - size_t position = 0, used = 0, output_length = length; - char *output; +static int jsonc_ascii_size(const char *data, size_t length, size_t *ascii_length, + char **error) { + size_t position = 0; + size_t output_length = length; while (position < length) { unsigned char first = (unsigned char)data[position]; - size_t start = position, encoded; - if (first < 0x80) { ++position; continue; } + size_t start = position; + size_t encoded; + if (first < 0x80) { + ++position; + continue; + } /* The preflight above guarantees successful scalar decoding. */ if (!dom_utf8(data, length, &position)) return dom_error(error, "Invalid JSON-C Unicode input"); encoded = first < 0xf0 ? 6 : 12; if (output_length > SIZE_MAX - encoded - 1) return dom_error(error, "JSON-C input size overflow"); output_length += encoded - (position - start); } - if (output_length == length) { *ascii = NULL; *ascii_length = length; return 0; } - output = (char *)malloc(output_length + 1); - if (output == NULL) return dom_error(error, "Unable to allocate JSON-C Unicode input"); - position = 0; + *ascii_length = output_length; + return 0; +} + +static unsigned int jsonc_next_codepoint(const char *data, size_t *position) { + unsigned int code = (unsigned char)data[*position]; + size_t count; + size_t i; + ++*position; + if (code < 0xe0) { + code &= 31; + count = 1; + } else if (code < 0xf0) { + code &= 15; + count = 2; + } else { + code &= 7; + count = 3; + } + for (i = 0; i < count; ++i) { + code = code * 64 + ((unsigned char)data[*position] & 63); + ++*position; + } + return code; +} + +static void jsonc_surrogate_escapes(char *output, size_t *used, unsigned int code) { + unsigned int offset = code - 0x10000; + jsonc_unicode_escape(output, used, 0xd800 + (offset >> 10)); + jsonc_unicode_escape(output, used, 0xdc00 + (offset & 1023)); +} + +static size_t jsonc_ascii_write(const char *data, size_t length, char *output) { + size_t position = 0; + size_t used = 0; while (position < length) { - unsigned int code = (unsigned char)data[position++]; - size_t count, i; - if (code < 0x80) { output[used++] = (char)code; continue; } - if (code < 0xe0) { code &= 31; count = 1; } - else if (code < 0xf0) { code &= 15; count = 2; } - else { code &= 7; count = 3; } - for (i = 0; i < count; ++i) code = code * 64 + ((unsigned char)data[position++] & 63); - if (code <= 0xffff) { - jsonc_unicode_escape(output, &used, code); - } else { - unsigned int offset = code - 0x10000; - jsonc_unicode_escape(output, &used, 0xd800 + (offset >> 10)); - jsonc_unicode_escape(output, &used, 0xdc00 + (offset & 1023)); + unsigned char first = (unsigned char)data[position]; + unsigned int code; + if (first < 0x80) { + output[used] = (char)first; + ++used; + ++position; + continue; } + code = jsonc_next_codepoint(data, &position); + if (code <= 0xffff) jsonc_unicode_escape(output, &used, code); + else jsonc_surrogate_escapes(output, &used, code); + } + return used; +} + +static int jsonc_ascii_input(const char *data, size_t length, char **ascii, + size_t *ascii_length, char **error) { + size_t output_length; + size_t used; + char *output; + int result = jsonc_ascii_size(data, length, &output_length, error); + if (result != 0) return result; + if (output_length == length) { + *ascii = NULL; + *ascii_length = length; + return 0; } + output = (char *)malloc(output_length + 1); + if (output == NULL) return dom_error(error, "Unable to allocate JSON-C Unicode input"); + used = jsonc_ascii_write(data, length, output); output[used] = '\0'; - *ascii = output; *ascii_length = used; + *ascii = output; + *ascii_length = used; return 0; } int jsonc_parse_file(jsonc_parser *parser, const char *filename, char **error_msg) { - char *content = NULL, *ascii = NULL; + char *content = NULL; + char *ascii = NULL; const char *input; - size_t length = 0, keys = 0, native_keys = 0, end; + size_t length = 0; + size_t keys = 0; + size_t native_keys = 0; + size_t end; dom_numbers numbers = {0}; dom_locale_guard locale = {0}; struct json_tokener *tokener = NULL; diff --git a/src/jsoncppparser.cpp b/src/jsoncppparser.cpp index 75731e6..8431d87 100644 --- a/src/jsoncppparser.cpp +++ b/src/jsoncppparser.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,11 @@ struct jsoncpp_parser { namespace { +class ParserError final : public std::runtime_error { + public: + using std::runtime_error::runtime_error; +}; + bool valid_limit(double value) { return std::isfinite(value) && value >= 0 && value <= UINT_MAX && std::floor(value) == value; @@ -35,60 +41,88 @@ int hex_digit(unsigned char c) { return -1; } +struct Utf8Sequence { + unsigned int code; + unsigned int continuations; + unsigned int minimum; +}; + +Utf8Sequence utf8_sequence(std::byte lead) { + if (lead >= std::byte{0xc2} && lead <= std::byte{0xdf}) + return {std::to_integer(lead & std::byte{0x1f}), 1, 0x80}; + if (lead >= std::byte{0xe0} && lead <= std::byte{0xef}) + return {std::to_integer(lead & std::byte{0x0f}), 2, 0x800}; + if (lead >= std::byte{0xf0} && lead <= std::byte{0xf4}) + return {std::to_integer(lead & std::byte{0x07}), 3, 0x10000}; + throw std::invalid_argument("Invalid UTF-8 in JSON input"); +} + +std::size_t utf8_end(const std::string& text, std::size_t start) { + auto sequence = utf8_sequence(static_cast(text[start])); + if (sequence.continuations > text.size() - start - 1) + throw std::invalid_argument("Truncated UTF-8 in JSON input"); + for (unsigned int n = 1; n <= sequence.continuations; ++n) { + const auto next = static_cast(text[start + n]); + if ((next & std::byte{0xc0}) != std::byte{0x80}) + throw std::invalid_argument("Invalid UTF-8 in JSON input"); + sequence.code = (sequence.code << 6) | + std::to_integer(next & std::byte{0x3f}); + } + if (sequence.code < sequence.minimum || sequence.code > 0x10ffff || + (sequence.code >= 0xd800 && sequence.code <= 0xdfff)) + throw std::invalid_argument("Invalid UTF-8 code point in JSON input"); + return start + sequence.continuations + 1; +} + +unsigned int unicode_unit(const std::string& text, std::size_t start) { + if (text.size() - start < 4) + throw std::invalid_argument("Truncated Unicode escape"); + unsigned int code = 0; + for (unsigned int n = 0; n < 4; ++n) { + const int digit = hex_digit(static_cast(text[start + n])); + if (digit < 0) throw std::invalid_argument("Invalid Unicode escape"); + code = (code << 4) | static_cast(digit); + } + return code; +} + +std::size_t escape_end(const std::string& text, std::size_t start) { + if (text.size() - start == 1) return text.size(); // Reader diagnoses truncation. + if (text[start + 1] != 'u') return start + 2; + const unsigned int first = unicode_unit(text, start + 2); + const std::size_t end = start + 6; + if (first >= 0xdc00 && first <= 0xdfff) + throw std::invalid_argument("Unpaired low Unicode surrogate"); + if (first < 0xd800 || first > 0xdbff) return end; + if (text.size() - end < 6 || text[end] != '\\' || text[end + 1] != 'u') + throw std::invalid_argument("Unpaired high Unicode surrogate"); + if (const unsigned int second = unicode_unit(text, end + 2); + second < 0xdc00 || second > 0xdfff) + throw std::invalid_argument("Invalid Unicode surrogate pair"); + return end + 6; +} + // JsonCpp accepts raw invalid UTF-8 and lone low surrogate escapes. Validate // these lexical boundaries before constructing its native DOM. The library // remains responsible for document grammar and decoded values. void validate_encoding(const std::string& text) { bool in_string = false; - for (std::size_t i = 0; i < text.size(); ++i) { - const unsigned char c = static_cast(text[i]); - if (c == 0) throw std::invalid_argument("Unexpected NUL in JSON input"); - if (in_string && c < 0x20) + std::size_t position = 0; + while (position < text.size()) { + const char c = text[position]; + if (c == '\0') throw std::invalid_argument("Unexpected NUL in JSON input"); + if (in_string && static_cast(c) < 0x20) throw std::invalid_argument("Unescaped control character in JSON string"); - if (c >= 0x80) { - unsigned int code = 0, count = 0, minimum = 0; - if (c >= 0xc2 && c <= 0xdf) { code = c & 0x1f; count = 1; minimum = 0x80; } - else if (c >= 0xe0 && c <= 0xef) { code = c & 0x0f; count = 2; minimum = 0x800; } - else if (c >= 0xf0 && c <= 0xf4) { code = c & 0x07; count = 3; minimum = 0x10000; } - else throw std::invalid_argument("Invalid UTF-8 in JSON input"); - if (count > text.size() - i - 1) - throw std::invalid_argument("Truncated UTF-8 in JSON input"); - for (unsigned int n = 0; n < count; ++n) { - const unsigned char next = static_cast(text[++i]); - if ((next & 0xc0) != 0x80) - throw std::invalid_argument("Invalid UTF-8 in JSON input"); - code = (code << 6) | (next & 0x3f); - } - if (code < minimum || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) - throw std::invalid_argument("Invalid UTF-8 code point in JSON input"); + if (static_cast(c) >= std::byte{0x80}) { + position = utf8_end(text, position); continue; } - if (c == '"') { in_string = !in_string; continue; } - if (!in_string || c != '\\') continue; - if (++i == text.size()) break; // The reader diagnoses truncation. - if (text[i] != 'u') continue; - auto unit = [&text](std::size_t at) { - if (text.size() - at < 4) - throw std::invalid_argument("Truncated Unicode escape"); - unsigned int code = 0; - for (unsigned int n = 0; n < 4; ++n) { - int digit = hex_digit(static_cast(text[at + n])); - if (digit < 0) throw std::invalid_argument("Invalid Unicode escape"); - code = (code << 4) | static_cast(digit); - } - return code; - }; - const unsigned int first = unit(i + 1); - i += 4; - if (first >= 0xdc00 && first <= 0xdfff) - throw std::invalid_argument("Unpaired low Unicode surrogate"); - if (first < 0xd800 || first > 0xdbff) continue; - if (text.size() - i - 1 < 6 || text[i + 1] != '\\' || text[i + 2] != 'u') - throw std::invalid_argument("Unpaired high Unicode surrogate"); - const unsigned int second = unit(i + 3); - if (second < 0xdc00 || second > 0xdfff) - throw std::invalid_argument("Invalid Unicode surrogate pair"); - i += 6; + if (c == '"') in_string = !in_string; + if (in_string && c == '\\') { + position = escape_end(text, position); + continue; + } + ++position; } } @@ -139,7 +173,58 @@ class Arguments { std::fwrite(": ", 1, 2, stdout) != 2 || std::fwrite(value.data(), 1, value.size(), stdout) != value.size() || std::fputc('\n', stdout) == EOF) - throw std::runtime_error("Unable to write JSON argument output"); + throw ParserError("Unable to write JSON argument output"); + } + + void visit_members(const Json::Value& value, unsigned int depth) { + // JsonCpp stores objects in key order. Source offsets restore input + // order for unique keys; duplicate keys are rejected by the reader. + auto names = value.getMemberNames(); + std::sort(names.begin(), names.end(), [&value](const std::string& a, const std::string& b) { + return value[a].getOffsetStart() < value[b].getOffsetStart(); + }); + for (const auto& key : names) { + path_.setKey(key.data(), key.size()); + check_path(); + visit(value[key], depth + 1); + } + } + + void visit_container(const Json::Value& value, unsigned int depth) { + if (depth >= settings_.max_depth) + throw std::length_error("JSON depth limit exceeded"); + if (value.isArray()) path_.startArraySpecial(); + else path_.push(); + check_path(); + if (value.isArray()) { + for (const auto& child : value) visit(child, depth + 1); + } else visit_members(value, depth); + path_.pop(); + } + + void visit_number(const Json::Value& value) { + const auto start = value.getOffsetStart(); + const auto end = value.getOffsetLimit(); + if (start < 0 || end < start || static_cast(end) > input_.size()) + throw ParserError("Invalid JsonCpp numeric source offsets"); + const std::string token = input_.substr(static_cast(start), + static_cast(end - start)); + if (!valid_number(token)) throw std::invalid_argument("Invalid JSON number"); + // Preserve the validated spelling, including integers beyond 64 bits, + // rather than converting its DOM double back to text. + emit(token); + } + + void visit_scalar(const Json::Value& value) { + switch (value.type()) { + case Json::nullValue: emit(""); break; + case Json::booleanValue: emit(value.asBool() ? "true" : "false"); break; + case Json::stringValue: emit(value.asString()); break; + case Json::intValue: + case Json::uintValue: + case Json::realValue: visit_number(value); break; + default: throw std::invalid_argument("Unsupported JSON value"); + } } public: @@ -147,45 +232,8 @@ class Arguments { : settings_(settings), input_(input) {} void visit(const Json::Value& value, unsigned int depth = 0) { - if (value.isObject() || value.isArray()) { - if (depth >= settings_.max_depth) - throw std::length_error("JSON depth limit exceeded"); - if (value.isArray()) path_.startArraySpecial(); - else path_.push(); - check_path(); - if (value.isArray()) { - for (const auto& child : value) visit(child, depth + 1); - } else { - // JsonCpp stores objects in key order. Source offsets restore - // input order for unique keys; duplicate keys are rejected. - auto names = value.getMemberNames(); - std::sort(names.begin(), names.end(), [&value](const std::string& a, const std::string& b) { - return value[a].getOffsetStart() < value[b].getOffsetStart(); - }); - for (const auto& key : names) { - path_.setKey(key.data(), key.size()); - check_path(); - visit(value[key], depth + 1); - } - } - path_.pop(); - return; - } - if (value.isNull()) emit(""); - else if (value.isBool()) emit(value.asBool() ? "true" : "false"); - else if (value.isString()) emit(value.asString()); - else if (value.isNumeric()) { - const auto start = value.getOffsetStart(); - const auto end = value.getOffsetLimit(); - if (start < 0 || end < start || static_cast(end) > input_.size()) - throw std::runtime_error("Invalid JsonCpp numeric source offsets"); - const std::string token = input_.substr(static_cast(start), - static_cast(end - start)); - if (!valid_number(token)) throw std::invalid_argument("Invalid JSON number"); - // Preserve the validated spelling, including integers beyond 64 - // bits, rather than converting its DOM double back to text. - emit(token); - } else throw std::invalid_argument("Unsupported JSON value"); + if (value.isObject() || value.isArray()) visit_container(value, depth); + else visit_scalar(value); } }; @@ -195,7 +243,7 @@ extern "C" int jsoncpp_json_init(jsoncpp_parser **parser, char **error_msg) { if (!parser || !error_msg) return -1; *parser = nullptr; *error_msg = nullptr; - try { *parser = new jsoncpp_parser{}; return 0; } + try { *parser = std::make_unique().release(); return 0; } catch (const std::exception& e) { *error_msg = strdup(e.what()); return 1; } catch (...) { *error_msg = strdup("Unable to initialize JsonCpp parser"); return 1; } } @@ -205,9 +253,9 @@ extern "C" int jsoncpp_parse_file(jsoncpp_parser *parser, const char *filename, if (!parser || !filename || !error_msg) return -1; try { std::ifstream file(filename, std::ios::binary); - if (!file.is_open()) throw std::runtime_error("Unable to open JSON file"); + if (!file.is_open()) throw ParserError("Unable to open JSON file"); const std::string input{std::istreambuf_iterator(file), std::istreambuf_iterator()}; - if (file.bad()) throw std::runtime_error("Unable to read JSON file"); + if (file.bad()) throw ParserError("Unable to read JSON file"); validate_encoding(input); Json::CharReaderBuilder builder; builder["collectComments"] = false; @@ -232,7 +280,7 @@ extern "C" int jsoncpp_parse_file(jsoncpp_parser *parser, const char *filename, throw std::invalid_argument(errors); Arguments(*parser, input).visit(root); if (!parser->silence && std::fflush(stdout) == EOF) - throw std::runtime_error("Unable to write JSON argument output"); + throw ParserError("Unable to write JSON argument output"); return 0; } catch (const std::exception& e) { *error_msg = strdup(e.what()); @@ -262,7 +310,7 @@ extern "C" int jsoncpp_set_silence(jsoncpp_parser *parser, int silence) { } extern "C" int jsoncpp_json_cleanup(jsoncpp_parser *parser) { - delete parser; + const std::unique_ptr owned(parser); return 0; } diff --git a/src/simdjsonparser.cpp b/src/simdjsonparser.cpp index 360dceb..3ae8415 100644 --- a/src/simdjsonparser.cpp +++ b/src/simdjsonparser.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,11 @@ struct simdjson_parser { namespace { +class ParserError final : public std::runtime_error { + public: + using std::runtime_error::runtime_error; +}; + bool valid_limit(double value) { return std::isfinite(value) && value >= 0 && value <= UINT_MAX && std::floor(value) == value; @@ -68,7 +74,7 @@ class Arguments { std::fwrite(": ", 1, 2, stdout) != 2 || std::fwrite(value.data(), 1, value.size(), stdout) != value.size() || std::fputc('\n', stdout) == EOF) - throw std::runtime_error("Unable to write JSON argument output"); + throw ParserError("Unable to write JSON argument output"); } public: @@ -156,7 +162,7 @@ extern "C" int simdjson_json_init(simdjson_parser **parser, char **error_msg) { if (!parser || !error_msg) return -1; *parser = nullptr; *error_msg = nullptr; - try { *parser = new simdjson_parser{}; return 0; } + try { *parser = std::make_unique().release(); return 0; } catch (const std::exception& e) { *error_msg = strdup(e.what()); return 1; } catch (...) { *error_msg = strdup("Unable to initialize simdjson parser"); return 1; } } @@ -176,7 +182,7 @@ extern "C" int simdjson_parse_file(simdjson_parser *parser, const char *filename if (!document.at_end()) throw std::invalid_argument("Unexpected content after JSON document"); if (!parser->silence && std::fflush(stdout) == EOF) - throw std::runtime_error("Unable to write JSON argument output"); + throw ParserError("Unable to write JSON argument output"); return 0; } catch (const std::exception& e) { *error_msg = strdup(e.what()); @@ -206,7 +212,7 @@ extern "C" int simdjson_set_silence(simdjson_parser *parser, int silence) { } extern "C" int simdjson_json_cleanup(simdjson_parser *parser) { - delete parser; + const std::unique_ptr owned(parser); return 0; } diff --git a/src/yyjsonparser.c b/src/yyjsonparser.c index 15ef504..9240c17 100644 --- a/src/yyjsonparser.c +++ b/src/yyjsonparser.c @@ -5,12 +5,14 @@ struct yyjson_parser { dom_settings settings; }; -static int yyjson_kind(const void *node) { - return yyjson_is_obj((yyjson_val *)node) ? DOM_OBJECT : yyjson_is_arr((yyjson_val *)node) ? DOM_ARRAY : DOM_SCALAR; +static int yyjson_kind(void *node) { + if (yyjson_is_obj((yyjson_val *)node)) return DOM_OBJECT; + if (yyjson_is_arr((yyjson_val *)node)) return DOM_ARRAY; + return DOM_SCALAR; } -static int yyjson_child(const void *node, dom_cursor *cursor, int first, - const void **child, const char **key, size_t *key_length) { +static int yyjson_child(void *node, dom_cursor *cursor, int first, + void **child, const char **key, size_t *key_length) { yyjson_val *json = (yyjson_val *)node; if (yyjson_is_arr(json)) { yyjson_arr_iter iter; @@ -33,7 +35,7 @@ static int yyjson_child(const void *node, dom_cursor *cursor, int first, return 1; } -static int yyjson_scalar(const void *node, dom_value *value, char **error) { +static int yyjson_scalar(void *node, dom_value *value, char **error) { yyjson_val *json = (yyjson_val *)node; if (yyjson_is_null(json)) { value->data = ""; value->length = 0; } else if (yyjson_is_bool(json)) { @@ -50,7 +52,8 @@ static const dom_adapter yyjson_adapter = {yyjson_kind, yyjson_child, yyjson_sca int yyjson_parse_file(yyjson_parser *parser, const char *filename, char **error_msg) { char *content = NULL; - size_t length = 0, keys = 0; + size_t length = 0; + size_t keys = 0; yyjson_read_err native_error; yyjson_doc *doc = NULL; int rc; diff --git a/tests/Makefile.am b/tests/Makefile.am index 22d5b77..2c8835b 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -8,6 +8,7 @@ AM_CXXFLAGS = -Wall -std=c++17 check_PROGRAMS = pathhandler_tests jsoncons_api_tests yajl_api_tests \ sax_adapter_api_tests dom_adapter_api_tests cpp_adapter_api_tests TESTS = $(check_PROGRAMS) +noinst_HEADERS = test_support.h pathhandler_tests_SOURCES = pathhandler_tests.cpp diff --git a/tests/cpp_adapter_api_tests.cpp b/tests/cpp_adapter_api_tests.cpp index e2ad290..8c83723 100644 --- a/tests/cpp_adapter_api_tests.cpp +++ b/tests/cpp_adapter_api_tests.cpp @@ -3,105 +3,28 @@ // numeric capability differences, and bounded traversal with enlarged limits. #include "../src/jsoncppparser.h" #include "../src/simdjsonparser.h" +#include "test_support.h" -#include +#include #include #include -#include +#include #include -#include -#include #include -#include -#include -#include +#include #if HAVE_JSONCPP || HAVE_SIMDJSON namespace { - -class Input { - std::string path_; - public: - explicit Input(const std::string& text) { - char name[] = "/tmp/jsonbench-cpp-api-XXXXXX"; - const int fd = mkstemp(name); - if (fd < 0) throw std::runtime_error("mkstemp failed"); - path_ = name; - std::size_t offset = 0; - while (offset < text.size()) { - const ssize_t count = write(fd, text.data() + offset, text.size() - offset); - if (count < 0 && errno == EINTR) continue; - if (count <= 0) { - close(fd); unlink(path_.c_str()); - throw std::runtime_error("temporary input write failed"); - } - offset += static_cast(count); - } - if (close(fd) != 0) { - unlink(path_.c_str()); - throw std::runtime_error("temporary input close failed"); - } - } - ~Input() { unlink(path_.c_str()); } - Input(const Input&) = delete; - Input& operator=(const Input&) = delete; - const char* path() const { return path_.c_str(); } -}; - -class Capture { - FILE* file_ = nullptr; - int saved_ = -1; - public: - Capture() { - if (std::fflush(stdout) != 0) throw std::runtime_error("stdout flush failed"); - file_ = std::tmpfile(); - if (!file_) throw std::runtime_error("output temporary file failed"); - saved_ = dup(STDOUT_FILENO); - if (saved_ < 0 || dup2(fileno(file_), STDOUT_FILENO) < 0) { - if (saved_ >= 0) close(saved_); - std::fclose(file_); - throw std::runtime_error("stdout capture failed"); - } - } - ~Capture() { - std::fflush(stdout); - dup2(saved_, STDOUT_FILENO); - close(saved_); - std::fclose(file_); - } - Capture(const Capture&) = delete; - Capture& operator=(const Capture&) = delete; - std::string read() { - if (std::fflush(stdout) != 0 || std::fseek(file_, 0, SEEK_SET) != 0) - throw std::runtime_error("output rewind failed"); - std::string output; - char buffer[1024]; - std::size_t count; - while ((count = std::fread(buffer, 1, sizeof buffer, file_)) != 0) - output.append(buffer, count); - if (std::ferror(file_)) throw std::runtime_error("output read failed"); - return output; - } -}; - -// The incoming diagnostic belongs to the caller. Successful parsing resets -// the slot without freeing it; failed parsing supplies a free()-owned message. -class ErrorSlot { - char* old_; - public: - char* value; - ErrorSlot() : old_(static_cast(std::malloc(1))), value(old_) { - if (!old_) throw std::bad_alloc(); - *old_ = '\0'; - } - ~ErrorSlot() { - if (value != old_) std::free(value); - std::free(old_); - } - ErrorSlot(const ErrorSlot&) = delete; - ErrorSlot& operator=(const ErrorSlot&) = delete; - bool replaced() const { return value != old_; } -}; +using namespace std::string_view_literals; + +using jsonbench::test::CapturedOutput; +using jsonbench::test::ErrorSlot; +using jsonbench::test::TemporaryInput; +using jsonbench::test::TestError; +using jsonbench::test::parse_path; +using jsonbench::test::parse_file; +template +using Parser = jsonbench::test::ApiParser; #define FILE_API(Type, Prefix, Label, DuplicateKeys, WideNumbers, SafeDepth) \ struct Type { \ @@ -126,85 +49,29 @@ FILE_API(SimdJsonApi, simdjson, "simdjson", true, false, 512) #endif #undef FILE_API -template -class Parser { - public: - typename Api::Handle* value = nullptr; - Parser() { - ErrorSlot error; - if (Api::init(&value, &error.value) != 0 || !value || error.value) { - Api::cleanup(value); - throw std::runtime_error(std::string(Api::name()) + " initialization failed"); - } - } - ~Parser() { Api::cleanup(value); } - Parser(const Parser&) = delete; - Parser& operator=(const Parser&) = delete; -}; - -struct Result { - int status; - bool replaced; - bool null_error; - std::string error; - std::string output; -}; +using Result = jsonbench::test::ParseResult; -template -Result parse_path(typename Api::Handle* parser, const char* path) { - ErrorSlot error; - Capture capture; - const int status = Api::file(parser, path, &error.value); - return {status, error.replaced(), !error.value, - error.value ? error.value : "", capture.read()}; -} - -template -Result parse(typename Api::Handle* parser, const std::string& text) { - Input input(text); - return parse_path(parser, input.path()); -} - -class Checks { - unsigned int passed_ = 0; - unsigned int failed_ = 0; +class Checks : public jsonbench::test::TestChecks { public: - std::string engine; - void check(const std::string& name, bool ok) { - if (ok) ++passed_; - else { - ++failed_; - std::fprintf(stderr, "FAIL: %s: %s\n", engine.c_str(), name.c_str()); - } - } - void success(const std::string& name, const Result& result, const std::string& expected = "") { - check(name + ": success and reset diagnostic", - result.status == 0 && result.replaced && result.null_error); - check(name + ": exact output", result.output == expected); - if (result.status != 0) std::fprintf(stderr, " diagnostic: %s\n", result.error.c_str()); + void success(std::string_view name, const Result& result, std::string_view expected = "") { + success2(name, result, expected); } - void failure(const std::string& name, const Result& result, const std::string& diagnostic = "") { - check(name + ": failure and owned diagnostic", result.status != 0 && - result.replaced && !result.null_error && !result.error.empty()); - if (!diagnostic.empty()) - check(name + ": diagnostic identifies limit", result.error.find(diagnostic) != std::string::npos); - } - int finish() const { - std::printf("C++ adapter C APIs: %u checks passed, %u failed\n", passed_, failed_); - return failed_ ? 1 : 0; + void failure(std::string_view name, const Result& result, std::string_view expected = "") { + failure1(name, result, expected); } + int finish() const { return report("C++ adapter C APIs"); } }; template void invalid_arguments(Checks& checks) { typename Api::Handle* handle = nullptr; ErrorSlot error; - checks.check("null initialization slots", Api::init(nullptr, &error.value) != 0 && + checks.check("null initialization slots", Api::init(nullptr, error.slot()) != 0 && Api::init(&handle, nullptr) != 0 && !handle && Api::init(nullptr, nullptr) != 0); checks.check("null setters and cleanup", Api::depth(nullptr, 1) != 0 && Api::arguments(nullptr, 1) != 0 && Api::silence(nullptr, 1) != 0 && Api::cleanup(nullptr) == 0); Parser parser; - Input input("true"); + TemporaryInput input("true"); const Result null_parser = parse_path(nullptr, input.path()); const Result null_filename = parse_path(parser.value, nullptr); checks.check("invalid parser/filename reset diagnostics and emit nothing", @@ -212,7 +79,7 @@ void invalid_arguments(Checks& checks) { null_filename.status != 0 && null_filename.replaced && null_filename.null_error && null_filename.output.empty()); checks.check("null diagnostic slot rejected", Api::file(parser.value, input.path(), nullptr) != 0); checks.failure("missing file", parse_path(parser.value, (std::string(input.path()) + ".missing").c_str())); - checks.success("reuse after invalid API calls", parse(parser.value, "true"), ": true\n"); + checks.success("reuse after invalid API calls", parse_file(parser.value, "true"), ": true\n"); } template @@ -220,62 +87,83 @@ void limits(Checks& checks) { Parser parser; checks.check("configure two containers and arguments", Api::depth(parser.value, 2) == 0 && Api::arguments(parser.value, 2) == 0 && Api::silence(parser.value, 1) == 0); - const std::vector invalid = {-1, -0.5, 0.5, 1.5, + const std::array invalid = {-1.0, -0.5, 0.5, 1.5, std::numeric_limits::quiet_NaN(), std::numeric_limits::infinity(), -std::numeric_limits::infinity(), static_cast(UINT_MAX) + 1}; for (std::size_t i = 0; i < invalid.size(); ++i) { const std::string label = "invalid setter " + std::to_string(i); checks.check(label, Api::depth(parser.value, invalid[i]) != 0 && Api::arguments(parser.value, invalid[i]) != 0); - checks.success(label + " preserves exact limits", parse(parser.value, "{\"a\":[true,null]}")); - checks.failure(label + " preserves depth ceiling", parse(parser.value, "[[[]]]")); - checks.failure(label + " preserves count ceiling", parse(parser.value, "[true,null,false]")); + checks.success(std::string(label) + " preserves exact limits", parse_file(parser.value, R"({"a":[true,null]})")); + checks.failure(std::string(label) + " preserves depth ceiling", parse_file(parser.value, "[[[]]]")); + checks.failure(std::string(label) + " preserves count ceiling", parse_file(parser.value, "[true,null,false]")); } checks.check("zero depth accepted", Api::depth(parser.value, -0.0) == 0); - for (const char* value : {"true", "null", "42", "\"value\""}) - checks.success(std::string("root scalar at depth zero ") + value, parse(parser.value, value)); + for (const char* value : {"true", "null", "42", R"("value")"}) + checks.success(std::string("root scalar at depth zero ") + value, parse_file(parser.value, value)); for (const char* value : {"{}", "[]"}) - checks.failure(std::string("container at depth zero ") + value, parse(parser.value, value), "depth"); + checks.failure(std::string("container at depth zero ") + value, parse_file(parser.value, value), "depth"); checks.check("zero count accepted", Api::depth(parser.value, 3) == 0 && Api::arguments(parser.value, 0) == 0); - checks.success("empty containers at count zero", parse(parser.value, "{\"a\":[{},[]]}")); - for (const char* value : {"true", "false", "null", "0", "\"\""}) - checks.failure(std::string("scalar at count zero ") + value, parse(parser.value, value), "Argument number"); + checks.success("empty containers at count zero", parse_file(parser.value, R"({"a":[{},[]]})")); + for (const char* value : {"true", "false", "null", "0", R"("")"}) + checks.failure(std::string("scalar at count zero ") + value, parse_file(parser.value, value), "Argument number"); checks.check("restore exact argument limit", Api::arguments(parser.value, 3) == 0); - checks.success("all scalar kinds counted exactly", parse(parser.value, "[[[null,true,false]]]")); - checks.failure("four scalar leaves exceed count", parse(parser.value, "[null,true,false,0]")); + checks.success("all scalar kinds counted exactly", parse_file(parser.value, "[[[null,true,false]]]")); + checks.failure("four scalar leaves exceed count", parse_file(parser.value, "[null,true,false,0]")); } template void reuse_and_numbers(Checks& checks) { Parser parser; - checks.success("source order and structural path restoration", parse(parser.value, - "{\"z\":[{\"b\":true},null],\"a.b\":{\"c\":2},\"after\":false}"), + checks.success("source order and structural path restoration", parse_file(parser.value, + R"({"z":[{"b":true},null],"a.b":{"c":2},"after":false})"), "z.z.b: true\nz.z: \na.b.c: 2\nafter: false\n"); - const char binary[] = "a\0b: x\0y\n"; - checks.success("embedded NUL name and value", parse(parser.value, "{\"a\\u0000b\":\"x\\u0000y\"}"), - std::string(binary, sizeof binary - 1)); - for (const char* invalid : {"[true,]", "1e+", "\"\\udc00\""}) { - checks.failure("syntax rejection", parse(parser.value, invalid)); - checks.success("clean state after syntax failure", parse(parser.value, "{\"fresh\":false}"), "fresh: false\n"); - } - const std::string finite = "{\"min\":-9223372036854775808,\"max\":9223372036854775807," - "\"u\":18446744073709551615,\"d\":0.123456789012345678901234567890,\"e\":1.2300e+03,\"n\":-0}"; - checks.success("lossless numeric spelling", parse(parser.value, finite), + constexpr auto binary = "a\0b: x\0y\n"sv; + checks.success("embedded NUL name and value", parse_file(parser.value, R"({"a\u0000b":"x\u0000y"})"), + std::string(binary)); + for (const char* invalid : {"[true,]", "1e+", R"("\udc00")"}) { + checks.failure("syntax rejection", parse_file(parser.value, invalid)); + checks.success("clean state after syntax failure", parse_file(parser.value, R"({"fresh":false})"), "fresh: false\n"); + } + const std::string finite = R"({"min":-9223372036854775808,"max":9223372036854775807,)" + R"("u":18446744073709551615,"d":0.123456789012345678901234567890,"e":1.2300e+03,"n":-0})"; + checks.success("lossless numeric spelling", parse_file(parser.value, finite), "min: -9223372036854775808\nmax: 9223372036854775807\nu: 18446744073709551615\n" "d: 0.123456789012345678901234567890\ne: 1.2300e+03\nn: -0\n"); for (const char* number : {"18446744073709551616", "-9223372036854775809", "1e9999"}) { if (Api::wide_numbers) - checks.success("raw numeric range retained", parse(parser.value, number), std::string(": ") + number + '\n'); - else checks.failure("unsupported native numeric range rejected", parse(parser.value, number)); + checks.success("raw numeric range retained", parse_file(parser.value, number), std::string(": ") + number + '\n'); + else checks.failure("unsupported native numeric range rejected", parse_file(parser.value, number)); } - const Result duplicate = parse(parser.value, "{\"k\":1,\"k\":2}"); + const Result duplicate = parse_file(parser.value, R"({"k":1,"k":2})"); if (Api::duplicates) checks.success("duplicate keys preserved", duplicate, "k: 1\nk: 2\n"); else checks.failure("duplicate keys explicitly rejected", duplicate, "Duplicate key"); checks.check("nonzero silence accepted", Api::silence(parser.value, -7) == 0); - checks.success("silence suppresses output", parse(parser.value, "{\"quiet\":true}")); - checks.failure("silence retains syntax validation", parse(parser.value, "{} []")); - checks.failure("silence rejects trailing raw NUL", parse(parser.value, std::string("{}\0", 3))); - checks.failure("oversized token rejected", parse(parser.value, "\"" + std::string(JSON_STRING_SIZE, 'x') + "\"")); - checks.success("reuse after native and resource failures", parse(parser.value, "[true,null]")); + checks.success("silence suppresses output", parse_file(parser.value, R"({"quiet":true})")); + checks.failure("silence retains syntax validation", parse_file(parser.value, "{} []")); + checks.failure("silence rejects trailing raw NUL", parse_file(parser.value, std::string("{}\0", 3))); + checks.failure("oversized token rejected", parse_file(parser.value, R"(")" + std::string(JSON_STRING_SIZE, 'x') + R"(")")); + checks.success("reuse after native and resource failures", parse_file(parser.value, "[true,null]")); +} + +std::string nested_array_argument(unsigned int depth) { + std::string output = "array"; + for (unsigned int i = 0; i < depth; ++i) output += ".array"; + return output + ": true\n"; +} + +template +void deep_case(Checks& checks, typename Api::Handle* parser, + unsigned int depth, bool leaf, bool silent) { + const std::string input = std::string(depth, '[') + (leaf ? "true" : "") + std::string(depth, ']'); + const std::string label = "depth " + std::to_string(depth) + + (leaf ? " boolean leaf" : " empty arrays") + (silent ? " silent" : " output"); + const Result result = parse_file(parser, input); + if (depth > Api::safe_depth) { + checks.failure(label, result, Api::duplicates ? "depth" : "stackLimit"); + return; + } + const std::string expected = leaf && !silent ? nested_array_argument(depth) : ""; + checks.success(label, result, expected); } template @@ -286,33 +174,21 @@ void large_depths(Checks& checks) { for (bool silent : {false, true}) { checks.check("set deep traversal mode", Api::silence(parser.value, silent) == 0); for (unsigned int depth : {500u, 501u, 512u, 513u, 999u, 2000u}) { - for (bool leaf : {false, true}) { - const std::string text = std::string(depth, '[') + (leaf ? "true" : "") + std::string(depth, ']'); - const std::string label = "depth " + std::to_string(depth) + - (leaf ? " boolean leaf" : " empty arrays") + (silent ? " silent" : " output"); - if (depth <= Api::safe_depth) { - std::string expected; - if (leaf && !silent) { - expected = "array"; - for (unsigned int i = 0; i < depth; ++i) expected += ".array"; - expected += ": true\n"; - } - checks.success(label, parse(parser.value, text), expected); - } else checks.failure(label, parse(parser.value, text), Api::duplicates ? "depth" : "stackLimit"); - } + for (bool leaf : {false, true}) + deep_case(checks, parser.value, depth, leaf, silent); } } checks.check("restore default depth and one argument", Api::depth(parser.value, 500) == 0 && Api::arguments(parser.value, 1) == 0 && Api::silence(parser.value, 1) == 0); - checks.success("default depth exact", parse(parser.value, std::string(500, '[') + "true" + std::string(500, ']'))); - checks.failure("default depth exceeded", parse(parser.value, std::string(501, '[') + "true" + std::string(501, ']'))); - checks.failure("boolean leaf consumes count", parse(parser.value, "[true,false]")); - checks.success("parser reusable after deep failures", parse(parser.value, "true")); + checks.success("default depth exact", parse_file(parser.value, std::string(500, '[') + "true" + std::string(500, ']'))); + checks.failure("default depth exceeded", parse_file(parser.value, std::string(501, '[') + "true" + std::string(501, ']'))); + checks.failure("boolean leaf consumes count", parse_file(parser.value, "[true,false]")); + checks.success("parser reusable after deep failures", parse_file(parser.value, "true")); } template void run(Checks& checks) { - checks.engine = Api::name(); + checks.set_engine(Api::name()); invalid_arguments(checks); limits(checks); reuse_and_numbers(checks); diff --git a/tests/dom_adapter_api_tests.c b/tests/dom_adapter_api_tests.c index 3f99c50..6630ca4 100644 --- a/tests/dom_adapter_api_tests.c +++ b/tests/dom_adapter_api_tests.c @@ -12,18 +12,32 @@ #include "../src/cjsonparser.h" #include "../src/yyjsonparser.h" static unsigned checks; -static char directory[] = "/tmp/jsonbench-dom-api-XXXXXX"; -static char valid[512], invalid[512], native_invalid[512], empty[512], root[512], decimal[512], missing[512]; +/* mkdtemp atomically creates this private directory with mode 0700. */ +static char directory[] = ".jsonbench-dom-api-XXXXXX"; +static int directory_created; +static char valid[512]; +static char invalid[512]; +static char native_invalid[512]; +static char empty[512]; +static char root[512]; +static char decimal[512]; +static char missing[512]; static void cleanup_fixtures(void) { + if (!directory_created) return; unlink(valid); unlink(invalid); unlink(native_invalid); unlink(empty); unlink(root); unlink(decimal); rmdir(directory); } static void fixture(char *path, size_t capacity, const char *name, const char *content) { FILE *file; + size_t written; + int close_result; int length = snprintf(path, capacity, "%s/%s.json", directory, name); if (length < 0 || (size_t)length >= capacity) exit(1); file = fopen(path, "wb"); - if (file == NULL || fwrite(content, 1, strlen(content), file) != strlen(content) || fclose(file) != 0) exit(1); + if (file == NULL) exit(1); + written = fwrite(content, 1, strlen(content), file); + close_result = fclose(file); + if (written != strlen(content) || close_result != 0) exit(1); } #define CHECK(condition) do { if (!(condition)) { fprintf(stderr,"FAIL line %d: %s\n",__LINE__,#condition); exit(1); } ++checks; } while (0) @@ -109,7 +123,8 @@ static void locale_checks(void) { static const char *names[] = {"de_DE.UTF-8", "de_DE.utf8", "fr_FR.UTF-8", "fr_FR.utf8"}; char *original = strdup(setlocale(LC_NUMERIC, NULL)); const char *selected = NULL; - locale_t caller_locale, previous; + locale_t caller_locale; + locale_t previous; CHECK(original != NULL); for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); ++i) { if (setlocale(LC_NUMERIC, names[i]) != NULL && strcmp(localeconv()->decimal_point, ",") == 0) { @@ -147,8 +162,9 @@ int main(void) { puts("SKIP DOM C API checks: no DOM engines enabled"); return 77; #endif - CHECK(mkdtemp(directory) != NULL); CHECK(atexit(cleanup_fixtures) == 0); + CHECK(mkdtemp(directory) != NULL); + directory_created = 1; fixture(valid, sizeof(valid), "valid", "{\"x\":[1,true,null,{\"k\":\"value\"}]}"); fixture(invalid, sizeof(invalid), "invalid", "{\"x\":\"\\q\"}"); fixture(native_invalid, sizeof(native_invalid), "native-invalid", "{\"a\":1,}"); From 46e8cdafbf616e22c18fbe2a03566f334d518de1 Mon Sep 17 00:00:00 2001 From: jens Steinbacher Date: Tue, 13 Oct 2026 17:10:37 +0000 Subject: [PATCH 07/18] Fix Sonar findings in CLI, YAJL and shared C++ test resources --- .sonarcloud.properties | 3 + src/jsonbench.c | 1238 +++++++--------------------------- src/jsonconsparser.cpp | 28 +- src/pathhandler.h | 5 +- src/yajlparser.c | 232 ++++--- tests/jsoncons_api_tests.cpp | 224 ++---- tests/pathhandler_tests.cpp | 47 +- tests/regression_tests.py | 2 + tests/test_support.h | 281 ++++++++ tests/yajl_api_tests.cpp | 358 ++++------ 10 files changed, 899 insertions(+), 1519 deletions(-) create mode 100644 .sonarcloud.properties create mode 100644 tests/test_support.h diff --git a/.sonarcloud.properties b/.sonarcloud.properties new file mode 100644 index 0000000..520ba92 --- /dev/null +++ b/.sonarcloud.properties @@ -0,0 +1,3 @@ +# Match the C++17 standard in src/Makefile.am and tests/Makefile.am. +# Automatic analysis otherwise recommends newer-standard facilities. +sonar.cfamily.reportingCppStandardOverride=c++17 diff --git a/src/jsonbench.c b/src/jsonbench.c index 7d70a2f..6f1317b 100644 --- a/src/jsonbench.c +++ b/src/jsonbench.c @@ -19,6 +19,7 @@ #include #include #include +#include #ifndef LIMIT_ARG_NUM #define LIMIT_ARG_NUM 500 @@ -55,1055 +56,322 @@ static void print_stats(const double *times, int n) { printf("\nTime: mean=%.9f sec stddev=%.9f sec (N=%d)\n\n", mean, stddev, n); } -static char available_engines[16][20] = {""}; -static int engine_count = 0; - -/* Do not let atoi overflow, accept a prefix, or turn negatives into unsigned - * limits. The parser interfaces take double, which exactly represents UINT_MAX. - */ -static int parse_limit(const char *text, unsigned int *value) { - if (text == NULL || *text == '\0') return 0; - for (const char *p = text; *p; ++p) { - if (*p < '0' || *p > '9') return 0; - } - errno = 0; - char *end = NULL; - unsigned long parsed = strtoul(text, &end, 10); - if (errno == ERANGE || *end != '\0' || parsed == 0 || parsed > UINT_MAX) - return 0; - *value = (unsigned int)parsed; - return 1; +typedef struct { + unsigned int depth_limit; + unsigned int arg_limit; + int silence; + int iterations; + const char *filename; +} benchmark_settings; + +typedef struct { + const char *name; + int (*parse)(const benchmark_settings *, int, int, char **); +} benchmark_engine; + +#if HAVE_YAJL || HAVE_RAPIDJSON || HAVE_NLOHMANNJSON || HAVE_JSONC || HAVE_JANSSON || HAVE_CJSON || HAVE_JSONCPP || HAVE_JSONCONS || HAVE_SIMDJSON || HAVE_YYJSON || HAVE_GLAZE +static int initialization_failed(char **error) { + if (*error == NULL) *error = strdup("Failed to initialize JSON parser"); + return EXIT_FAILURE; } -static void showhelp(void) { - printf("Use: jsonbench [OPTIONS]\n\n"); - printf("OPTIONS:\n"); - printf("\t-h\tThis help\n"); - printf("\t-d\tSet maximum depth of JSON structure, default: %d\n", LIMIT_DEPTH); - printf("\t-a\tSet maximum number of possible ARGS, default: %d\n", LIMIT_ARG_NUM); - printf("\t-n\tNumber of timed iterations, default: 1\n"); - printf("\t-s\tBe silence; don't print out the parsed data\n"); - if (engine_count > 0) { - printf("\t-e\tUse JSON engine\n"); - printf("\t \tavailable engines:\n"); - for(int i = 0; i < engine_count; i++) { - printf("\t \t- %s\n", available_engines[i]); - } - } - else { - printf("No JSON engine available\n"); +#endif + +#if HAVE_RAPIDJSON || HAVE_NLOHMANNJSON || HAVE_JSONC || HAVE_JANSSON || HAVE_CJSON || HAVE_JSONCPP || HAVE_JSONCONS || HAVE_SIMDJSON || HAVE_YYJSON || HAVE_GLAZE +static int check_limit_result(int result, const char *engine, const char *limit, + int report_support, char **error) { + if (result == 0) return 0; + if (result == 1) { + if (report_support) + fprintf(stderr, "%s does not support max %s limit\n", engine, limit); + return 0; } - printf("\n"); + if (*error == NULL) *error = strdup("Failed to configure JSON parser limit"); + return EXIT_FAILURE; } -int main(int argc, char ** argv) { - char c; - char *jsonengine = NULL; - - extern char *optarg; - extern int optind, opterr, optopt; +#endif - char *error_msg = NULL; - const char *jsonfile = NULL; - unsigned int depth_limit = LIMIT_DEPTH; - unsigned int arg_limit = LIMIT_ARG_NUM; - int silence = 0; - int n = 1; - struct timespec ts_before, ts_after, ts_diff; +/* Keep the native C parser pointer types: casting incompatible init/cleanup + * function pointers would be undefined behavior. Only the common lifecycle is + * generated here; all grammar/path work remains in the existing adapters. */ +#define DEFINE_ENGINE_RUN(prefix, parser_type, engine_name) \ +static int run_##prefix(const benchmark_settings *settings, int silence, \ + int probe, char **error) { \ + parser_type *parser = NULL; \ + int result = prefix##_json_init(&parser, error); \ + if (result != 0 || parser == NULL) { \ + prefix##_json_cleanup(parser); \ + return initialization_failed(error); \ + } \ + result = check_limit_result(prefix##_set_max_depth(parser, settings->depth_limit), \ + engine_name, "depth", probe, error); \ + if (result == 0) \ + result = check_limit_result(prefix##_set_max_arg_num(parser, settings->arg_limit), \ + engine_name, "arg num", probe, error); \ + if (result == 0) result = prefix##_set_silence(parser, silence); \ + if (result == 0 && !probe) \ + result = prefix##_parse_file(parser, settings->filename, error); \ + prefix##_json_cleanup(parser); \ + return result; \ +} #if HAVE_YAJL - strcpy(available_engines[engine_count++], "YAJL"); +static int run_yajl(const benchmark_settings *settings, int silence, + int probe, char **error) { + yajl_json_data *parser = NULL; + const int initialized = yajl_json_init(&parser, error); + if (initialized != 1 || parser == NULL) { + yajl_json_cleanup(parser); + return initialization_failed(error); + } + parser->depth_limit = settings->depth_limit; + parser->arg_num_limit = settings->arg_limit; + parser->silence = silence; + int result = 0; + if (!probe) result = yajl_parse_file(parser, settings->filename, error); + yajl_json_cleanup(parser); + return result; +} #endif #if HAVE_RAPIDJSON - strcpy(available_engines[engine_count++], "RAPIDJSON"); +DEFINE_ENGINE_RUN(rj, rj_parser, "RAPIDJSON") #endif #if HAVE_NLOHMANNJSON - strcpy(available_engines[engine_count++], "NLOHMANNJSON"); +DEFINE_ENGINE_RUN(nl, nl_parser, "NLOHMANNJSON") #endif #if HAVE_JSONC - strcpy(available_engines[engine_count++], "JSONC"); +DEFINE_ENGINE_RUN(jsonc, jsonc_parser, "JSONC") #endif #if HAVE_JANSSON - strcpy(available_engines[engine_count++], "JANSSON"); +DEFINE_ENGINE_RUN(jansson, jansson_parser, "JANSSON") #endif #if HAVE_CJSON - strcpy(available_engines[engine_count++], "CJSON"); +DEFINE_ENGINE_RUN(cjson, cjson_parser, "CJSON") #endif #if HAVE_JSONCPP - strcpy(available_engines[engine_count++], "JSONCPP"); +DEFINE_ENGINE_RUN(jsoncpp, jsoncpp_parser, "JSONCPP") #endif #if HAVE_JSONCONS - strcpy(available_engines[engine_count++], "JSONCONS"); +DEFINE_ENGINE_RUN(jsoncons, jsoncons_parser, "JSONCONS") #endif #if HAVE_SIMDJSON - strcpy(available_engines[engine_count++], "SIMDJSON"); +DEFINE_ENGINE_RUN(simdjson, simdjson_parser, "SIMDJSON") #endif #if HAVE_YYJSON - strcpy(available_engines[engine_count++], "YYJSON"); +DEFINE_ENGINE_RUN(yyjson, yyjson_parser, "YYJSON") #endif #if HAVE_GLAZE - strcpy(available_engines[engine_count++], "GLAZE"); +DEFINE_ENGINE_RUN(glaze, glaze_parser, "GLAZE") #endif +#undef DEFINE_ENGINE_RUN - while ((c = getopt(argc, argv, "he:a:d:n:s")) != -1) { - switch (c) { - case 'h': - showhelp(); - return 0; - case 'e': - jsonengine = strdup(optarg); - if (jsonengine == NULL) { - fprintf(stderr, "Memory allocation error\n"); - return EXIT_FAILURE; - } - - int i = engine_count; - for(int e = 0; e < engine_count; e++) { - if (strcmp(jsonengine, available_engines[e]) == 0) { - i = e; - break; - } - } - - if (i == engine_count) { - fprintf(stderr, "JSON engine '%s' is not available!\n", jsonengine); - free(jsonengine); - return EXIT_FAILURE; - } - break; - case 'a': - if (!parse_limit(optarg, &arg_limit)) { - fprintf(stderr, "Pass for '-a' an integer between 1 and %u\n", - (unsigned int)UINT_MAX); - free(jsonengine); - return EXIT_FAILURE; - } - break; - case 'd': - if (!parse_limit(optarg, &depth_limit)) { - fprintf(stderr, "Pass for '-d' an integer between 1 and %u\n", - (unsigned int)UINT_MAX); - free(jsonengine); - return EXIT_FAILURE; - } - break; - case 'n': - n = atoi(optarg); - if (n <= 0) { - fprintf(stderr, "'-n' must be a positive integer\n"); - free(jsonengine); - return EXIT_FAILURE; - } - break; - case 's': - silence = 1; - break; - default: - showhelp(); - free(jsonengine); - return 0; - } - } - - if (jsonengine == NULL) { - printf("No JSON engine was given!\n"); - return 0; - } - - for (int i = optind; i < argc; i++) { - if (jsonfile == NULL) { - jsonfile = argv[i]; - } - } - - if (jsonfile == NULL) { - printf("No JSON file was given!\n"); - free(jsonengine); - return EXIT_FAILURE; - } - - /* When running multiple iterations, silence output during timed passes to - * avoid measuring printf overhead. A final non-timed pass prints output - * if the user did not request silence. */ - int timed_silence = (n > 1) ? 1 : silence; - int rc = 0; - - double *times = malloc((size_t)n * sizeof(double)); - if (times == NULL) { - fprintf(stderr, "Memory allocation error\n"); - free(jsonengine); - return EXIT_FAILURE; - } - +static const benchmark_engine engines[] = { #if HAVE_YAJL - if (strcmp(jsonengine, "YAJL") == 0) { - for (int iter = 0; iter < n; iter++) { - yajl_json_data *json = NULL; - - clock_gettime(CLOCK_MONOTONIC, &ts_before); - - yajl_json_init(&json, &error_msg); - if (json == NULL) { - fprintf(stderr, "Failed to initialize JSON parser\n"); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - json->depth_limit = depth_limit; - json->arg_num_limit = arg_limit; - json->silence = timed_silence; - - rc = yajl_parse_file(json, jsonfile, &error_msg); - yajl_json_cleanup(json); - - clock_gettime(CLOCK_MONOTONIC, &ts_after); - - if (rc != 0) { - if (error_msg) { - fprintf(stderr, "Error: %s\n", error_msg); - free(error_msg); - error_msg = NULL; - } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - - timespec_diff(&ts_after, &ts_before, &ts_diff); - times[iter] = ts_diff.tv_sec + ts_diff.tv_nsec * 1e-9; - } - - print_stats(times, n); - - if (n > 1 && !silence) { - yajl_json_data *json = NULL; - rc = 1; - yajl_json_init(&json, &error_msg); - if (json != NULL) { - json->depth_limit = depth_limit; - json->arg_num_limit = arg_limit; - json->silence = 0; - rc = yajl_parse_file(json, jsonfile, &error_msg); - yajl_json_cleanup(json); - } - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - } - } + {"YAJL", run_yajl}, #endif - #if HAVE_RAPIDJSON - if (strcmp(jsonengine, "RAPIDJSON") == 0) { - for (int iter = 0; iter < n; iter++) { - rj_parser *json = NULL; - - clock_gettime(CLOCK_MONOTONIC, &ts_before); - - rj_json_init(&json, &error_msg); - if (json == NULL) { - fprintf(stderr, "Failed to initialize JSON parser\n"); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - rj_set_max_depth(json, depth_limit); - rj_set_max_arg_num(json, arg_limit); - rj_set_silence(json, timed_silence); - - rc = rj_parse_file(json, jsonfile, &error_msg); - rj_json_cleanup(json); - - clock_gettime(CLOCK_MONOTONIC, &ts_after); - - if (rc != 0) { - if (error_msg) { - fprintf(stderr, "Error: %s\n", error_msg); - free(error_msg); - error_msg = NULL; - } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - - timespec_diff(&ts_after, &ts_before, &ts_diff); - times[iter] = ts_diff.tv_sec + ts_diff.tv_nsec * 1e-9; - } - - print_stats(times, n); - - if (n > 1 && !silence) { - rj_parser *json = NULL; - rc = 1; - rj_json_init(&json, &error_msg); - if (json != NULL) { - rj_set_max_depth(json, depth_limit); - rj_set_max_arg_num(json, arg_limit); - rj_set_silence(json, 0); - rc = rj_parse_file(json, jsonfile, &error_msg); - rj_json_cleanup(json); - } - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - } - } + {"RAPIDJSON", run_rj}, #endif - #if HAVE_NLOHMANNJSON - if (strcmp(jsonengine, "NLOHMANNJSON") == 0) { - for (int iter = 0; iter < n; iter++) { - nl_parser *json = NULL; - - clock_gettime(CLOCK_MONOTONIC, &ts_before); - - nl_json_init(&json, &error_msg); - if (json == NULL) { - fprintf(stderr, "Failed to initialize JSON parser\n"); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - nl_set_max_depth(json, depth_limit); - nl_set_max_arg_num(json, arg_limit); - nl_set_silence(json, timed_silence); - - rc = nl_parse_file(json, jsonfile, &error_msg); - nl_json_cleanup(json); - - clock_gettime(CLOCK_MONOTONIC, &ts_after); - - if (rc != 0) { - if (error_msg) { - fprintf(stderr, "Error: %s\n", error_msg); - free(error_msg); - error_msg = NULL; - } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - - timespec_diff(&ts_after, &ts_before, &ts_diff); - times[iter] = ts_diff.tv_sec + ts_diff.tv_nsec * 1e-9; - } - - print_stats(times, n); - - if (n > 1 && !silence) { - nl_parser *json = NULL; - rc = 1; - nl_json_init(&json, &error_msg); - if (json != NULL) { - nl_set_max_depth(json, depth_limit); - nl_set_max_arg_num(json, arg_limit); - nl_set_silence(json, 0); - rc = nl_parse_file(json, jsonfile, &error_msg); - nl_json_cleanup(json); - } - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - } - } + {"NLOHMANNJSON", run_nl}, #endif - #if HAVE_JSONC - if (strcmp(jsonengine, "JSONC") == 0) { - /* Probe limit support once (untimed) before the measurement loop */ - { - jsonc_parser *probe = NULL; - char *probe_err = NULL; - if (jsonc_json_init(&probe, &probe_err) == 0) { - if (jsonc_set_max_depth(probe, depth_limit) == 1) - fprintf(stderr, "JSONC does not support max depth limit\n"); - if (jsonc_set_max_arg_num(probe, arg_limit) == 1) - fprintf(stderr, "JSONC does not support max arg num limit\n"); - jsonc_json_cleanup(probe); - } - if (probe_err) free(probe_err); - } - for (int iter = 0; iter < n; iter++) { - jsonc_parser *json = NULL; - - clock_gettime(CLOCK_MONOTONIC, &ts_before); - - rc = jsonc_json_init(&json, &error_msg); - if (rc != 0 || json == NULL) { - fprintf(stderr, "Failed to initialize JSON parser\n"); - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); error_msg = NULL; } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - rc = jsonc_set_max_depth(json, depth_limit); - if (rc != 0 && rc != 1) { - fprintf(stderr, "jsonc_set_max_depth failed with code %d\n", rc); - jsonc_json_cleanup(json); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - rc = jsonc_set_max_arg_num(json, arg_limit); - if (rc != 0 && rc != 1) { - fprintf(stderr, "jsonc_set_max_arg_num failed with code %d\n", rc); - jsonc_json_cleanup(json); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - jsonc_set_silence(json, timed_silence); - - rc = jsonc_parse_file(json, jsonfile, &error_msg); - jsonc_json_cleanup(json); - - clock_gettime(CLOCK_MONOTONIC, &ts_after); - - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); error_msg = NULL; } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - - timespec_diff(&ts_after, &ts_before, &ts_diff); - times[iter] = ts_diff.tv_sec + ts_diff.tv_nsec * 1e-9; - } - - print_stats(times, n); - - if (n > 1 && !silence) { - jsonc_parser *json = NULL; - rc = 1; - jsonc_json_init(&json, &error_msg); - if (json != NULL) { - jsonc_set_max_depth(json, depth_limit); - jsonc_set_max_arg_num(json, arg_limit); - jsonc_set_silence(json, 0); - rc = jsonc_parse_file(json, jsonfile, &error_msg); - jsonc_json_cleanup(json); - } - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - } - } + {"JSONC", run_jsonc}, #endif - #if HAVE_JANSSON - if (strcmp(jsonengine, "JANSSON") == 0) { - /* Probe limit support once (untimed) before the measurement loop */ - { - jansson_parser *probe = NULL; - char *probe_err = NULL; - if (jansson_json_init(&probe, &probe_err) == 0) { - if (jansson_set_max_depth(probe, depth_limit) == 1) - fprintf(stderr, "JANSSON does not support max depth limit\n"); - if (jansson_set_max_arg_num(probe, arg_limit) == 1) - fprintf(stderr, "JANSSON does not support max arg num limit\n"); - jansson_json_cleanup(probe); - } - if (probe_err) free(probe_err); - } - for (int iter = 0; iter < n; iter++) { - jansson_parser *json = NULL; - - clock_gettime(CLOCK_MONOTONIC, &ts_before); - - rc = jansson_json_init(&json, &error_msg); - if (rc != 0 || json == NULL) { - fprintf(stderr, "Failed to initialize JSON parser\n"); - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); error_msg = NULL; } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - rc = jansson_set_max_depth(json, depth_limit); - if (rc != 0 && rc != 1) { - fprintf(stderr, "jansson_set_max_depth failed with code %d\n", rc); - jansson_json_cleanup(json); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - rc = jansson_set_max_arg_num(json, arg_limit); - if (rc != 0 && rc != 1) { - fprintf(stderr, "jansson_set_max_arg_num failed with code %d\n", rc); - jansson_json_cleanup(json); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - jansson_set_silence(json, timed_silence); - - rc = jansson_parse_file(json, jsonfile, &error_msg); - jansson_json_cleanup(json); - - clock_gettime(CLOCK_MONOTONIC, &ts_after); - - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); error_msg = NULL; } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - - timespec_diff(&ts_after, &ts_before, &ts_diff); - times[iter] = ts_diff.tv_sec + ts_diff.tv_nsec * 1e-9; - } - - print_stats(times, n); - - if (n > 1 && !silence) { - jansson_parser *json = NULL; - rc = 1; - jansson_json_init(&json, &error_msg); - if (json != NULL) { - jansson_set_max_depth(json, depth_limit); - jansson_set_max_arg_num(json, arg_limit); - jansson_set_silence(json, 0); - rc = jansson_parse_file(json, jsonfile, &error_msg); - jansson_json_cleanup(json); - } - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - } - } + {"JANSSON", run_jansson}, #endif - #if HAVE_CJSON - if (strcmp(jsonengine, "CJSON") == 0) { - /* Probe limit support once (untimed) before the measurement loop */ - { - cjson_parser *probe = NULL; - char *probe_err = NULL; - if (cjson_json_init(&probe, &probe_err) == 0) { - if (cjson_set_max_depth(probe, depth_limit) == 1) - fprintf(stderr, "CJSON does not support max depth limit\n"); - if (cjson_set_max_arg_num(probe, arg_limit) == 1) - fprintf(stderr, "CJSON does not support max arg num limit\n"); - cjson_json_cleanup(probe); - } - if (probe_err) free(probe_err); - } - for (int iter = 0; iter < n; iter++) { - cjson_parser *json = NULL; - - clock_gettime(CLOCK_MONOTONIC, &ts_before); - - rc = cjson_json_init(&json, &error_msg); - if (rc != 0 || json == NULL) { - fprintf(stderr, "Failed to initialize JSON parser\n"); - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); error_msg = NULL; } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - rc = cjson_set_max_depth(json, depth_limit); - if (rc != 0 && rc != 1) { - fprintf(stderr, "cjson_set_max_depth failed with code %d\n", rc); - cjson_json_cleanup(json); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - rc = cjson_set_max_arg_num(json, arg_limit); - if (rc != 0 && rc != 1) { - fprintf(stderr, "cjson_set_max_arg_num failed with code %d\n", rc); - cjson_json_cleanup(json); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - cjson_set_silence(json, timed_silence); - - rc = cjson_parse_file(json, jsonfile, &error_msg); - cjson_json_cleanup(json); - - clock_gettime(CLOCK_MONOTONIC, &ts_after); - - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); error_msg = NULL; } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - - timespec_diff(&ts_after, &ts_before, &ts_diff); - times[iter] = ts_diff.tv_sec + ts_diff.tv_nsec * 1e-9; - } - - print_stats(times, n); - - if (n > 1 && !silence) { - cjson_parser *json = NULL; - rc = 1; - cjson_json_init(&json, &error_msg); - if (json != NULL) { - cjson_set_max_depth(json, depth_limit); - cjson_set_max_arg_num(json, arg_limit); - cjson_set_silence(json, 0); - rc = cjson_parse_file(json, jsonfile, &error_msg); - cjson_json_cleanup(json); - } - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - } - } + {"CJSON", run_cjson}, #endif - #if HAVE_JSONCPP - if (strcmp(jsonengine, "JSONCPP") == 0) { - /* Probe limit support once (untimed) before the measurement loop */ - { - jsoncpp_parser *probe = NULL; - char *probe_err = NULL; - if (jsoncpp_json_init(&probe, &probe_err) == 0) { - if (jsoncpp_set_max_depth(probe, depth_limit) == 1) - fprintf(stderr, "JSONCPP does not support max depth limit\n"); - if (jsoncpp_set_max_arg_num(probe, arg_limit) == 1) - fprintf(stderr, "JSONCPP does not support max arg num limit\n"); - jsoncpp_json_cleanup(probe); - } - if (probe_err) free(probe_err); - } - for (int iter = 0; iter < n; iter++) { - jsoncpp_parser *json = NULL; - - clock_gettime(CLOCK_MONOTONIC, &ts_before); - - rc = jsoncpp_json_init(&json, &error_msg); - if (rc != 0 || json == NULL) { - fprintf(stderr, "Failed to initialize JSON parser\n"); - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); error_msg = NULL; } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - rc = jsoncpp_set_max_depth(json, depth_limit); - if (rc != 0 && rc != 1) { - fprintf(stderr, "jsoncpp_set_max_depth failed with code %d\n", rc); - jsoncpp_json_cleanup(json); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - rc = jsoncpp_set_max_arg_num(json, arg_limit); - if (rc != 0 && rc != 1) { - fprintf(stderr, "jsoncpp_set_max_arg_num failed with code %d\n", rc); - jsoncpp_json_cleanup(json); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - jsoncpp_set_silence(json, timed_silence); - - rc = jsoncpp_parse_file(json, jsonfile, &error_msg); - jsoncpp_json_cleanup(json); - - clock_gettime(CLOCK_MONOTONIC, &ts_after); - - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); error_msg = NULL; } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - - timespec_diff(&ts_after, &ts_before, &ts_diff); - times[iter] = ts_diff.tv_sec + ts_diff.tv_nsec * 1e-9; - } - - print_stats(times, n); - - if (n > 1 && !silence) { - jsoncpp_parser *json = NULL; - rc = 1; - jsoncpp_json_init(&json, &error_msg); - if (json != NULL) { - jsoncpp_set_max_depth(json, depth_limit); - jsoncpp_set_max_arg_num(json, arg_limit); - jsoncpp_set_silence(json, 0); - rc = jsoncpp_parse_file(json, jsonfile, &error_msg); - jsoncpp_json_cleanup(json); - } - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - } - } + {"JSONCPP", run_jsoncpp}, #endif - #if HAVE_JSONCONS - if (strcmp(jsonengine, "JSONCONS") == 0) { - /* Probe limit support once (untimed) before the measurement loop */ - { - jsoncons_parser *probe = NULL; - char *probe_err = NULL; - if (jsoncons_json_init(&probe, &probe_err) == 0) { - if (jsoncons_set_max_depth(probe, depth_limit) == 1) - fprintf(stderr, "JSONCONS does not support max depth limit\n"); - if (jsoncons_set_max_arg_num(probe, arg_limit) == 1) - fprintf(stderr, "JSONCONS does not support max arg num limit\n"); - jsoncons_json_cleanup(probe); - } - if (probe_err) free(probe_err); - } - for (int iter = 0; iter < n; iter++) { - jsoncons_parser *json = NULL; - - clock_gettime(CLOCK_MONOTONIC, &ts_before); - - rc = jsoncons_json_init(&json, &error_msg); - if (rc != 0 || json == NULL) { - fprintf(stderr, "Failed to initialize JSON parser\n"); - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); error_msg = NULL; } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - rc = jsoncons_set_max_depth(json, depth_limit); - if (rc != 0 && rc != 1) { - fprintf(stderr, "jsoncons_set_max_depth failed with code %d\n", rc); - jsoncons_json_cleanup(json); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - rc = jsoncons_set_max_arg_num(json, arg_limit); - if (rc != 0 && rc != 1) { - fprintf(stderr, "jsoncons_set_max_arg_num failed with code %d\n", rc); - jsoncons_json_cleanup(json); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - jsoncons_set_silence(json, timed_silence); - - rc = jsoncons_parse_file(json, jsonfile, &error_msg); - jsoncons_json_cleanup(json); - - clock_gettime(CLOCK_MONOTONIC, &ts_after); - - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); error_msg = NULL; } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - - timespec_diff(&ts_after, &ts_before, &ts_diff); - times[iter] = ts_diff.tv_sec + ts_diff.tv_nsec * 1e-9; - } - - print_stats(times, n); - - if (n > 1 && !silence) { - jsoncons_parser *json = NULL; - rc = jsoncons_json_init(&json, &error_msg); - if (rc == 0 && json != NULL) { - jsoncons_set_max_depth(json, depth_limit); - jsoncons_set_max_arg_num(json, arg_limit); - jsoncons_set_silence(json, 0); - rc = jsoncons_parse_file(json, jsonfile, &error_msg); - jsoncons_json_cleanup(json); - } else { - rc = 1; - } - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - } - } + {"JSONCONS", run_jsoncons}, #endif - #if HAVE_SIMDJSON - if (strcmp(jsonengine, "SIMDJSON") == 0) { - /* Probe limit support once (untimed) before the measurement loop */ - { - simdjson_parser *probe = NULL; - char *probe_err = NULL; - if (simdjson_json_init(&probe, &probe_err) == 0) { - if (simdjson_set_max_depth(probe, depth_limit) == 1) - fprintf(stderr, "SIMDJSON does not support max depth limit\n"); - if (simdjson_set_max_arg_num(probe, arg_limit) == 1) - fprintf(stderr, "SIMDJSON does not support max arg num limit\n"); - simdjson_json_cleanup(probe); - } - if (probe_err) free(probe_err); - } - for (int iter = 0; iter < n; iter++) { - simdjson_parser *json = NULL; - - clock_gettime(CLOCK_MONOTONIC, &ts_before); - - rc = simdjson_json_init(&json, &error_msg); - if (rc != 0 || json == NULL) { - fprintf(stderr, "Failed to initialize JSON parser\n"); - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); error_msg = NULL; } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - rc = simdjson_set_max_depth(json, depth_limit); - if (rc != 0 && rc != 1) { - fprintf(stderr, "simdjson_set_max_depth failed with code %d\n", rc); - simdjson_json_cleanup(json); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - rc = simdjson_set_max_arg_num(json, arg_limit); - if (rc != 0 && rc != 1) { - fprintf(stderr, "simdjson_set_max_arg_num failed with code %d\n", rc); - simdjson_json_cleanup(json); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - simdjson_set_silence(json, timed_silence); - - rc = simdjson_parse_file(json, jsonfile, &error_msg); - simdjson_json_cleanup(json); - - clock_gettime(CLOCK_MONOTONIC, &ts_after); - - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); error_msg = NULL; } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - - timespec_diff(&ts_after, &ts_before, &ts_diff); - times[iter] = ts_diff.tv_sec + ts_diff.tv_nsec * 1e-9; - } - - print_stats(times, n); - - if (n > 1 && !silence) { - simdjson_parser *json = NULL; - rc = 1; - simdjson_json_init(&json, &error_msg); - if (json != NULL) { - simdjson_set_max_depth(json, depth_limit); - simdjson_set_max_arg_num(json, arg_limit); - simdjson_set_silence(json, 0); - rc = simdjson_parse_file(json, jsonfile, &error_msg); - simdjson_json_cleanup(json); - } - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - } - } + {"SIMDJSON", run_simdjson}, #endif - #if HAVE_YYJSON - if (strcmp(jsonengine, "YYJSON") == 0) { - /* Probe limit support once (untimed) before the measurement loop */ - { - yyjson_parser *probe = NULL; - char *probe_err = NULL; - if (yyjson_json_init(&probe, &probe_err) == 0) { - if (yyjson_set_max_depth(probe, depth_limit) == 1) - fprintf(stderr, "YYJSON does not support max depth limit\n"); - if (yyjson_set_max_arg_num(probe, arg_limit) == 1) - fprintf(stderr, "YYJSON does not support max arg num limit\n"); - yyjson_json_cleanup(probe); - } - if (probe_err) free(probe_err); - } - for (int iter = 0; iter < n; iter++) { - yyjson_parser *json = NULL; - - clock_gettime(CLOCK_MONOTONIC, &ts_before); - - rc = yyjson_json_init(&json, &error_msg); - if (rc != 0 || json == NULL) { - fprintf(stderr, "Failed to initialize JSON parser\n"); - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); error_msg = NULL; } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - rc = yyjson_set_max_depth(json, depth_limit); - if (rc != 0 && rc != 1) { - fprintf(stderr, "yyjson_set_max_depth failed with code %d\n", rc); - yyjson_json_cleanup(json); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - rc = yyjson_set_max_arg_num(json, arg_limit); - if (rc != 0 && rc != 1) { - fprintf(stderr, "yyjson_set_max_arg_num failed with code %d\n", rc); - yyjson_json_cleanup(json); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - yyjson_set_silence(json, timed_silence); - - rc = yyjson_parse_file(json, jsonfile, &error_msg); - yyjson_json_cleanup(json); - - clock_gettime(CLOCK_MONOTONIC, &ts_after); - - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); error_msg = NULL; } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - - timespec_diff(&ts_after, &ts_before, &ts_diff); - times[iter] = ts_diff.tv_sec + ts_diff.tv_nsec * 1e-9; - } - - print_stats(times, n); - - if (n > 1 && !silence) { - yyjson_parser *json = NULL; - rc = 1; - yyjson_json_init(&json, &error_msg); - if (json != NULL) { - yyjson_set_max_depth(json, depth_limit); - yyjson_set_max_arg_num(json, arg_limit); - yyjson_set_silence(json, 0); - rc = yyjson_parse_file(json, jsonfile, &error_msg); - yyjson_json_cleanup(json); - } - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - } - } + {"YYJSON", run_yyjson}, #endif - #if HAVE_GLAZE - if (strcmp(jsonengine, "GLAZE") == 0) { - /* Probe limit support once (untimed) before the measurement loop */ - { - glaze_parser *probe = NULL; - char *probe_err = NULL; - if (glaze_json_init(&probe, &probe_err) == 0) { - if (glaze_set_max_depth(probe, depth_limit) == 1) - fprintf(stderr, "GLAZE does not support max depth limit\n"); - if (glaze_set_max_arg_num(probe, arg_limit) == 1) - fprintf(stderr, "GLAZE does not support max arg num limit\n"); - glaze_json_cleanup(probe); - } - if (probe_err) free(probe_err); - } - for (int iter = 0; iter < n; iter++) { - glaze_parser *json = NULL; + {"GLAZE", run_glaze}, +#endif + {NULL, NULL} +}; - clock_gettime(CLOCK_MONOTONIC, &ts_before); +/* Do not accept prefixes/negative values or overflow the unsigned parser + * limits. Parser double parameters represent every UINT_MAX value exactly. */ +static int parse_limit(const char *text, unsigned int *value) { + if (text == NULL || *text == '\0') return 0; + for (const char *cursor = text; *cursor; ++cursor) { + if (*cursor < '0' || *cursor > '9') return 0; + } + errno = 0; + char *end = NULL; + const unsigned long parsed = strtoul(text, &end, 10); + if (errno == ERANGE || *end != '\0' || parsed == 0 || parsed > UINT_MAX) + return 0; + *value = (unsigned int)parsed; + return 1; +} - rc = glaze_json_init(&json, &error_msg); - if (rc != 0 || json == NULL) { - fprintf(stderr, "Failed to initialize JSON parser\n"); - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); error_msg = NULL; } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - rc = glaze_set_max_depth(json, depth_limit); - if (rc != 0 && rc != 1) { - fprintf(stderr, "glaze_set_max_depth failed with code %d\n", rc); - glaze_json_cleanup(json); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - rc = glaze_set_max_arg_num(json, arg_limit); - if (rc != 0 && rc != 1) { - fprintf(stderr, "glaze_set_max_arg_num failed with code %d\n", rc); - glaze_json_cleanup(json); - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - glaze_set_silence(json, timed_silence); +static void showhelp(void) { + printf("Use: jsonbench [OPTIONS]\n\n"); + printf("OPTIONS:\n"); + printf("\t-h\tThis help\n"); + printf("\t-d\tSet maximum depth of JSON structure, default: %d\n", LIMIT_DEPTH); + printf("\t-a\tSet maximum number of possible ARGS, default: %d\n", LIMIT_ARG_NUM); + printf("\t-n\tNumber of timed iterations, default: 1\n"); + printf("\t-s\tBe silence; don't print out the parsed data\n"); + if (engines[0].name == NULL) { + printf("No JSON engine available\n\n"); + return; + } + printf("\t-e\tUse JSON engine\n\t \tavailable engines:\n"); + for (const benchmark_engine *engine = engines; engine->name; ++engine) + printf("\t \t- %s\n", engine->name); + printf("\n"); +} - rc = glaze_parse_file(json, jsonfile, &error_msg); - glaze_json_cleanup(json); +static const benchmark_engine *find_engine(const char *name) { + for (const benchmark_engine *engine = engines; engine->name; ++engine) { + if (strcmp(name, engine->name) == 0) return engine; + } + return NULL; +} - clock_gettime(CLOCK_MONOTONIC, &ts_after); +static int accept_limit_option(int option, const char *argument, + benchmark_settings *settings) { + unsigned int *limit = &settings->arg_limit; + if (option == 'd') limit = &settings->depth_limit; + if (parse_limit(argument, limit)) return 0; + fprintf(stderr, "Pass for '-%c' an integer between 1 and %u\n", option, UINT_MAX); + return -1; +} - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); error_msg = NULL; } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } +static int accept_option(int option, const char *argument, + benchmark_settings *settings, + const benchmark_engine **engine) { + unsigned int iterations = 0; + switch (option) { + case 'h': + showhelp(); + return 1; + case 'e': + *engine = find_engine(argument); + if (*engine != NULL) return 0; + fprintf(stderr, "JSON engine '%s' is not available!\n", argument); + return -1; + case 'a': + case 'd': + return accept_limit_option(option, argument, settings); + case 'n': + if (!parse_limit(argument, &iterations) || iterations > INT_MAX) { + fprintf(stderr, "'-n' must be a positive integer no greater than %d\n", INT_MAX); + return -1; + } + settings->iterations = (int)iterations; + return 0; + case 's': + settings->silence = 1; + return 0; + default: + showhelp(); + return 1; + } +} - timespec_diff(&ts_after, &ts_before, &ts_diff); - times[iter] = ts_diff.tv_sec + ts_diff.tv_nsec * 1e-9; - } +static int parse_options(int argc, char **argv, benchmark_settings *settings, + const benchmark_engine **engine) { + int option; + while ((option = getopt(argc, argv, "he:a:d:n:s")) != -1) { + const int result = accept_option(option, optarg, settings, engine); + if (result != 0) return result; + } + if (*engine == NULL) { + printf("No JSON engine was given!\n"); + return 1; + } + if (optind == argc) { + printf("No JSON file was given!\n"); + return -1; + } + settings->filename = argv[optind]; + return 0; +} - print_stats(times, n); +static int report_parser_result(int result, char *error) { + if (result != 0) + fprintf(stderr, "Error: %s\n", error ? error : "JSON parser failed"); + free(error); + return result != 0; +} - if (n > 1 && !silence) { - glaze_parser *json = NULL; - rc = 1; - glaze_json_init(&json, &error_msg); - if (json != NULL) { - glaze_set_max_depth(json, depth_limit); - glaze_set_max_arg_num(json, arg_limit); - glaze_set_silence(json, 0); - rc = glaze_parse_file(json, jsonfile, &error_msg); - glaze_json_cleanup(json); - } - if (rc != 0) { - if (error_msg) { fprintf(stderr, "Error: %s\n", error_msg); free(error_msg); } - free(times); - free(jsonengine); - return EXIT_FAILURE; - } - } +static int parse_once(const benchmark_engine *engine, + const benchmark_settings *settings, int silence, + int probe, double *elapsed) { + struct timespec before; + struct timespec after; + struct timespec difference; + if (!probe && clock_gettime(CLOCK_MONOTONIC, &before) != 0) { + fprintf(stderr, "Unable to read benchmark clock\n"); + return EXIT_FAILURE; } -#endif + char *error = NULL; + const int result = engine->parse(settings, silence, probe, &error); + if (report_parser_result(result, error)) return EXIT_FAILURE; + if (probe) return 0; + if (clock_gettime(CLOCK_MONOTONIC, &after) != 0) { + fprintf(stderr, "Unable to read benchmark clock\n"); + return EXIT_FAILURE; + } + timespec_diff(&after, &before, &difference); + *elapsed = difference.tv_sec + difference.tv_nsec * 1e-9; + return 0; +} - free(times); - free(jsonengine); +static int benchmark(const benchmark_engine *engine, + const benchmark_settings *settings, double *times) { + if (parse_once(engine, settings, 1, 1, NULL) != 0) return EXIT_FAILURE; + int timed_silence = settings->silence; + if (settings->iterations > 1) timed_silence = 1; + for (int iteration = 0; iteration < settings->iterations; ++iteration) { + if (parse_once(engine, settings, timed_silence, 0, ×[iteration]) != 0) + return EXIT_FAILURE; + } + print_stats(times, settings->iterations); + if (settings->iterations > 1 && !settings->silence) { + double elapsed; + if (parse_once(engine, settings, 0, 0, &elapsed) != 0) return EXIT_FAILURE; + } if (fflush(stdout) != 0) { fprintf(stderr, "Error: Unable to write benchmark output\n"); return EXIT_FAILURE; } return 0; } + +int main(int argc, char **argv) { + benchmark_settings settings = {LIMIT_DEPTH, LIMIT_ARG_NUM, 0, 1, NULL}; + const benchmark_engine *engine = NULL; + const int parsed = parse_options(argc, argv, &settings, &engine); + if (parsed < 0) return EXIT_FAILURE; + if (parsed > 0) return 0; + const size_t count = (size_t)settings.iterations; + if (count > SIZE_MAX / sizeof(double)) { + fprintf(stderr, "Benchmark iteration allocation size overflow\n"); + return EXIT_FAILURE; + } + double *times = malloc(count * sizeof(double)); + if (times == NULL) { + fprintf(stderr, "Memory allocation error\n"); + return EXIT_FAILURE; + } + const int result = benchmark(engine, &settings, times); + free(times); + return result; +} diff --git a/src/jsonconsparser.cpp b/src/jsonconsparser.cpp index 34e7fe6..00436f0 100644 --- a/src/jsonconsparser.cpp +++ b/src/jsonconsparser.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -28,6 +29,11 @@ struct jsoncons_parser { namespace { +class UnicodeError final : public std::invalid_argument { + public: + using std::invalid_argument::invalid_argument; +}; + // jsoncons 1.3.2 silently drops lone escaped low surrogates and does not check // that the second code unit of a pair is a low surrogate. Validate just this // lexical boundary while forwarding its public stream source, including pairs @@ -43,7 +49,7 @@ class UnicodeSource { std::size_t column_ = 0; [[noreturn]] void invalid() const { - throw std::invalid_argument("Invalid Unicode surrogate escape at line " + + throw UnicodeError("Invalid Unicode surrogate escape at line " + std::to_string(line_) + ", column " + std::to_string(column_)); } @@ -137,10 +143,8 @@ class TemporaryAllocator { template bool operator!=(const TemporaryAllocator&) const { return false; } }; -// Keep structural boundaries rather than finding them by searching for dots. -// PathHandler cannot distinguish literal dots/empty keys from nesting, and its -// fixed buffers have unsafe boundary checks. This stack follows the same usual -// dotted naming convention without copying those limitations. +// Keep structural boundaries explicitly so dots, empty keys and NULs remain +// key data while argument names follow the historical dotted convention. class ArgumentVisitor final : public jsoncons::default_json_visitor { struct Frame { std::size_t prefix_length; @@ -166,8 +170,9 @@ class ArgumentVisitor final : public jsoncons::default_json_visitor { return; } const bool root_array = array && stack_.empty(); - const std::size_t length = root_array ? 5 : - prefix_.size() + (prefix_.empty() ? 0 : 1) + key_.size(); + std::size_t length = prefix_.size() + key_.size(); + if (!prefix_.empty()) ++length; + if (root_array) length = 5; if (length >= JSON_STRING_SIZE) { fail("Argument name too long", ec); return; @@ -309,9 +314,9 @@ extern "C" int jsoncons_json_init(jsoncons_parser** parser, char** error_msg) { if (parser == nullptr) return -1; *parser = nullptr; try { - *parser = new jsoncons_parser{}; + *parser = std::make_unique().release(); return 0; - } catch (const std::exception& e) { + } catch (const std::bad_alloc& e) { *error_msg = strdup(e.what()); return 1; } catch (...) { @@ -362,6 +367,9 @@ extern "C" int jsoncons_parse_file(jsoncons_parser* parser, const char* filename return 2; } return 0; + } catch (const jsoncons::json_exception& e) { + *error_msg = strdup(e.what()); + return 2; } catch (const std::exception& e) { *error_msg = strdup(e.what()); return 2; @@ -383,7 +391,7 @@ extern "C" int jsoncons_set_silence(jsoncons_parser* parser, int silence) { return 0; } extern "C" int jsoncons_json_cleanup(jsoncons_parser* parser) { - delete parser; + const std::unique_ptr owned(parser); return 0; } diff --git a/src/pathhandler.h b/src/pathhandler.h index 55c7e4b..b08796c 100644 --- a/src/pathhandler.h +++ b/src/pathhandler.h @@ -25,8 +25,9 @@ class PathHandler { void start(bool array) { if (m_error) return; const bool root_array = array && frames_.empty(); - const std::size_t length = root_array ? 5 : - prefix_.size() + (prefix_.empty() ? 0 : 1) + key_.size(); + std::size_t length = 5; + if (!root_array) + length = prefix_.size() + (prefix_.empty() ? 0 : 1) + key_.size(); if (length >= JSON_STRING_SIZE) { m_error = true; return; diff --git a/src/yajlparser.c b/src/yajlparser.c index c2a1dfd..59ee5bf 100644 --- a/src/yajlparser.c +++ b/src/yajlparser.c @@ -65,8 +65,9 @@ static int start_container(yajl_json_data *p, int array) { return reject(p, "JSON depth limit exceeded"); } int root_array = array && p->frame_count == 0; - size_t length = root_array ? 5 : - p->prefix_len + (p->prefix_len ? 1 : 0) + p->current_key_len; + size_t length = 5; + if (!root_array) + length = p->prefix_len + (p->prefix_len ? 1 : 0) + p->current_key_len; if (length >= JSON_STRING_SIZE) return reject(p, "Argument name too long"); if (p->frame_count == p->frame_capacity) { if (p->frame_capacity > SIZE_MAX / (2 * sizeof(yajl_path_frame))) @@ -85,7 +86,8 @@ static int start_container(yajl_json_data *p, int array) { if (root_array) { memcpy(p->prefix, "array", 5); memcpy(p->current_key, "array", 5); - p->prefix_len = p->current_key_len = 5; + p->prefix_len = 5; + p->current_key_len = 5; } else { if (p->prefix_len) p->prefix[p->prefix_len++] = '.'; memcpy(p->prefix + p->prefix_len, p->current_key, p->current_key_len); @@ -118,13 +120,21 @@ static const yajl_callbacks callbacks = { static void clear_state(yajl_json_data *p) { for (size_t i = 0; i < p->frame_count; ++i) free(p->frames[i].key); p->frame_count = 0; - p->prefix_len = p->current_key_len = 0; - p->prefix[0] = p->current_key[0] = '\0'; - p->current_depth = p->current_arg_num = 0; - p->depth_limit_exceeded = p->arg_num_limit_exceeded = 0; + p->prefix_len = 0; + p->current_key_len = 0; + p->prefix[0] = '\0'; + p->current_key[0] = '\0'; + p->current_depth = 0; + p->current_arg_num = 0; + p->depth_limit_exceeded = 0; + p->arg_num_limit_exceeded = 0; p->callback_error = NULL; - p->escape_state = p->escape_digits = p->escape_codepoint = 0; - p->low_surrogate_required = p->number_token = p->completed = 0; + p->escape_state = 0; + p->escape_digits = 0; + p->escape_codepoint = 0; + p->low_surrogate_required = 0; + p->number_token = 0; + p->completed = 0; p->token_length = 0; p->utf8_remaining = 0; p->utf8_min = 0x80; @@ -152,7 +162,8 @@ int yajl_json_init(yajl_json_data **json, char **error_msg) { if (!p) { *error_msg = strdup("Unable to allocate YAJL parser"); return -1; } p->prefix = malloc(JSON_STRING_SIZE); p->current_key = malloc(JSON_STRING_SIZE); - p->depth_limit = p->arg_num_limit = 500; + p->depth_limit = 500; + p->arg_num_limit = 500; if (!p->prefix || !p->current_key || reset(p, error_msg) < 0) { if (!*error_msg) *error_msg = strdup("Unable to allocate YAJL path buffers"); yajl_json_cleanup(p); @@ -165,92 +176,131 @@ int yajl_json_init(yajl_json_data **json, char **error_msg) { // Narrow incremental token/Unicode guard. YAJL 2.1 does not validate that an // escaped surrogate pair's second unit is a low surrogate. It also grows lexer // buffers before callbacks can reject oversized values. JSON grammar stays YAJL's. +static int check_utf8_byte(yajl_json_data *p, unsigned char c) { + // YAJL 2.1's native check accepts overlong encodings and surrogate + // codepoints; enforce Unicode scalar UTF-8 independently across chunks. + if (p->utf8_remaining) { + if (c < p->utf8_min || c > p->utf8_max) + return reject(p, "Invalid UTF-8 sequence"); + --p->utf8_remaining; + p->utf8_min = 0x80; + p->utf8_max = 0xbf; + return 1; + } + if (c < 0x80) return 1; + p->utf8_min = 0x80; + p->utf8_max = 0xbf; + if (c >= 0xc2 && c <= 0xdf) { + p->utf8_remaining = 1; + return 1; + } + if (c >= 0xe0 && c <= 0xef) { + p->utf8_remaining = 2; + if (c == 0xe0) p->utf8_min = 0xa0; + if (c == 0xed) p->utf8_max = 0x9f; + return 1; + } + if (c >= 0xf0 && c <= 0xf4) { + p->utf8_remaining = 3; + if (c == 0xf0) p->utf8_min = 0x90; + if (c == 0xf4) p->utf8_max = 0x8f; + return 1; + } + return reject(p, "Invalid UTF-8 sequence"); +} + +static void start_unicode_escape(yajl_json_data *p) { + p->escape_state = 3; + p->escape_digits = 0; + p->escape_codepoint = 0; +} + +static int finish_unicode_escape(yajl_json_data *p) { + if (p->low_surrogate_required) { + if (p->escape_codepoint < 0xdc00 || p->escape_codepoint > 0xdfff) + return reject(p, "Invalid Unicode surrogate escape"); + p->low_surrogate_required = 0; + p->escape_state = 1; + return 1; + } + if (p->escape_codepoint >= 0xd800 && p->escape_codepoint <= 0xdbff) { + p->low_surrogate_required = 1; + p->escape_state = 4; + return 1; + } + if (p->escape_codepoint >= 0xdc00 && p->escape_codepoint <= 0xdfff) + return reject(p, "Invalid Unicode surrogate escape"); + p->escape_state = 1; + return 1; +} + +static int check_unicode_digit(yajl_json_data *p, unsigned char c) { + unsigned int digit; + if (c >= '0' && c <= '9') digit = c - '0'; + else if (c >= 'a' && c <= 'f') digit = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') digit = c - 'A' + 10; + else return reject(p, "Invalid Unicode escape"); + p->escape_codepoint = p->escape_codepoint * 16 + digit; + ++p->escape_digits; + if (p->escape_digits < 4) return 1; + return finish_unicode_escape(p); +} + +static int check_escape_byte(yajl_json_data *p, unsigned char c) { + switch (p->escape_state) { + case 1: + if (c == '"') { + p->escape_state = 0; + p->token_length = 0; + } else if (c == '\\') p->escape_state = 2; + break; + case 2: + if (c == 'u') start_unicode_escape(p); + else p->escape_state = 1; + break; + case 3: + return check_unicode_digit(p, c); + case 4: + if (c != '\\') return reject(p, "Invalid Unicode surrogate escape"); + p->escape_state = 5; + break; + case 5: + if (c != 'u') return reject(p, "Invalid Unicode surrogate escape"); + start_unicode_escape(p); + break; + } + return 1; +} + +static int check_token_byte(yajl_json_data *p, unsigned char c, size_t budget) { + if (p->escape_state) { + ++p->token_length; + if (p->token_length > budget) return reject(p, "JSON token allocation limit exceeded"); + return check_escape_byte(p, c); + } + if (c == '"') { + p->number_token = 0; + p->escape_state = 1; + p->token_length = 1; + return 1; + } + int number_char = (c >= '0' && c <= '9') || c == '-' || c == '+' || c == '.' || c == 'e' || c == 'E'; + if (p->number_token && number_char) { + ++p->token_length; + if (p->token_length > budget) return reject(p, "JSON token allocation limit exceeded"); + } else { + p->number_token = (c >= '0' && c <= '9') || c == '-'; + p->token_length = p->number_token ? 1 : 0; + } + return 1; +} + static int check_input(yajl_json_data *p, const char *buf, size_t length) { const size_t budget = (size_t)JSON_STRING_SIZE * 8 + 16; for (size_t i = 0; i < length; ++i) { unsigned char c = (unsigned char)buf[i]; if (!c) return reject(p, "Invalid raw NUL byte in JSON input"); - // YAJL 2.1's native check accepts overlong encodings and surrogate - // codepoints; enforce Unicode scalar UTF-8 independently across chunks. - if (p->utf8_remaining) { - if (c < p->utf8_min || c > p->utf8_max) - return reject(p, "Invalid UTF-8 sequence"); - --p->utf8_remaining; - p->utf8_min = 0x80; - p->utf8_max = 0xbf; - } else if (c >= 0x80) { - p->utf8_min = 0x80; - p->utf8_max = 0xbf; - if (c >= 0xc2 && c <= 0xdf) p->utf8_remaining = 1; - else if (c >= 0xe0 && c <= 0xef) { - p->utf8_remaining = 2; - if (c == 0xe0) p->utf8_min = 0xa0; - if (c == 0xed) p->utf8_max = 0x9f; - } else if (c >= 0xf0 && c <= 0xf4) { - p->utf8_remaining = 3; - if (c == 0xf0) p->utf8_min = 0x90; - if (c == 0xf4) p->utf8_max = 0x8f; - } else return reject(p, "Invalid UTF-8 sequence"); - } - if (p->escape_state) { - if (++p->token_length > budget) return reject(p, "JSON token allocation limit exceeded"); - switch (p->escape_state) { - case 1: - if (c == '"') { p->escape_state = 0; p->token_length = 0; } - else if (c == '\\') p->escape_state = 2; - break; - case 2: - if (c == 'u') { - p->escape_state = 3; - p->escape_digits = p->escape_codepoint = 0; - } else p->escape_state = 1; - break; - case 3: { - unsigned int digit; - if (c >= '0' && c <= '9') digit = c - '0'; - else if (c >= 'a' && c <= 'f') digit = c - 'a' + 10; - else if (c >= 'A' && c <= 'F') digit = c - 'A' + 10; - else return reject(p, "Invalid Unicode escape"); - p->escape_codepoint = p->escape_codepoint * 16 + digit; - if (++p->escape_digits < 4) break; - if (p->low_surrogate_required) { - if (p->escape_codepoint < 0xdc00 || p->escape_codepoint > 0xdfff) - return reject(p, "Invalid Unicode surrogate escape"); - p->low_surrogate_required = 0; - p->escape_state = 1; - } else if (p->escape_codepoint >= 0xd800 && p->escape_codepoint <= 0xdbff) { - p->low_surrogate_required = 1; - p->escape_state = 4; - } else { - if (p->escape_codepoint >= 0xdc00 && p->escape_codepoint <= 0xdfff) - return reject(p, "Invalid Unicode surrogate escape"); - p->escape_state = 1; - } - break; - } - case 4: - if (c != '\\') return reject(p, "Invalid Unicode surrogate escape"); - p->escape_state = 5; - break; - case 5: - if (c != 'u') return reject(p, "Invalid Unicode surrogate escape"); - p->escape_state = 3; - p->escape_digits = p->escape_codepoint = 0; - break; - } - } else if (c == '"') { - p->number_token = 0; - p->escape_state = 1; - p->token_length = 1; - } else { - int number_char = (c >= '0' && c <= '9') || c == '-' || c == '+' || c == '.' || c == 'e' || c == 'E'; - if (p->number_token && number_char) { - if (++p->token_length > budget) return reject(p, "JSON token allocation limit exceeded"); - } else { - p->number_token = (c >= '0' && c <= '9') || c == '-'; - p->token_length = p->number_token ? 1 : 0; - } - } + if (!check_utf8_byte(p, c) || !check_token_byte(p, c, budget)) return 0; } return 1; } diff --git a/tests/jsoncons_api_tests.cpp b/tests/jsoncons_api_tests.cpp index e474f1e..b53d127 100644 --- a/tests/jsoncons_api_tests.cpp +++ b/tests/jsoncons_api_tests.cpp @@ -1,123 +1,34 @@ // Direct regression checks for the C interface, including parser reuse and // limits that cannot be represented by the CLI's positive-integer options. #include "../src/jsonconsparser.h" +#include "test_support.h" -#include #include -#include #include -#include -#include +#include #include -#include -#include #include +#include #include #include -#include - #if HAVE_JSONCONS namespace { -class TemporaryInput { - std::string path_; - - public: - explicit TemporaryInput(const std::string& text) { - char name[] = "/tmp/jsonbench-jsoncons-api-XXXXXX"; - const int fd = mkstemp(name); - if (fd < 0) throw std::runtime_error("mkstemp failed"); - path_ = name; - std::size_t written = 0; - while (written < text.size()) { - const ssize_t count = write(fd, text.data() + written, text.size() - written); - if (count < 0 && errno == EINTR) continue; - if (count <= 0) { - close(fd); - unlink(path_.c_str()); - throw std::runtime_error("temporary input write failed"); - } - written += static_cast(count); - } - if (close(fd) != 0) { - unlink(path_.c_str()); - throw std::runtime_error("temporary input close failed"); - } - } - ~TemporaryInput() { unlink(path_.c_str()); } - TemporaryInput(const TemporaryInput&) = delete; - TemporaryInput& operator=(const TemporaryInput&) = delete; - const char* path() const { return path_.c_str(); } -}; - -class CapturedOutput { - FILE* file_ = nullptr; - int saved_fd_ = -1; - - public: - CapturedOutput() { - if (std::fflush(stdout) != 0) throw std::runtime_error("stdout flush failed"); - file_ = std::tmpfile(); - if (file_ == nullptr) throw std::runtime_error("stdout temporary file failed"); - saved_fd_ = dup(STDOUT_FILENO); - if (saved_fd_ < 0 || dup2(fileno(file_), STDOUT_FILENO) < 0) { - if (saved_fd_ >= 0) close(saved_fd_); - std::fclose(file_); - throw std::runtime_error("stdout capture failed"); - } - } - ~CapturedOutput() { - std::fflush(stdout); - dup2(saved_fd_, STDOUT_FILENO); - close(saved_fd_); - std::fclose(file_); - } - CapturedOutput(const CapturedOutput&) = delete; - CapturedOutput& operator=(const CapturedOutput&) = delete; - std::string read() { - if (std::fflush(stdout) != 0 || std::fseek(file_, 0, SEEK_SET) != 0) - throw std::runtime_error("captured output rewind failed"); - std::string text; - char buffer[1024]; - std::size_t count; - while ((count = std::fread(buffer, 1, sizeof buffer, file_)) != 0) - text.append(buffer, count); - if (std::ferror(file_)) throw std::runtime_error("captured output read failed"); - return text; - } -}; - -// Start each error slot with a caller-owned allocation. The API must reset -// the pointer, not free the caller's old value. Free returned diagnostics with -// free(), as required by the C interface, on success and failure paths alike. -class ErrorSlot { - char* original_; - - public: - char* value; - ErrorSlot() : original_(static_cast(std::malloc(1))), value(original_) { - if (original_ == nullptr) throw std::bad_alloc(); - *original_ = '\0'; - } - ~ErrorSlot() { - if (value != original_) std::free(value); - std::free(original_); - } - ErrorSlot(const ErrorSlot&) = delete; - ErrorSlot& operator=(const ErrorSlot&) = delete; - bool replaced() const { return value != original_; } -}; +using jsonbench::test::CapturedOutput; +using jsonbench::test::ErrorSlot; +using jsonbench::test::TemporaryInput; +using jsonbench::test::TestError; class Parser { public: jsoncons_parser* value = nullptr; Parser() { ErrorSlot error; - const int rc = jsoncons_json_init(&value, &error.value); - if (rc != 0 || value == nullptr || error.value != nullptr) { + const int rc = jsoncons_json_init(&value, error.slot()); + if (rc != 0 || value == nullptr || error.get() != nullptr) { jsoncons_json_cleanup(value); - throw std::runtime_error("parser initialization failed"); + throw TestError("parser initialization failed"); } } ~Parser() { jsoncons_json_cleanup(value); } @@ -125,64 +36,37 @@ class Parser { Parser& operator=(const Parser&) = delete; }; -struct ParseResult { - int status; - bool error_replaced; - bool error_is_null; - std::string error; - std::string output; -}; +using ParseResult = jsonbench::test::ParseResult; ParseResult parse_path(jsoncons_parser* parser, const char* path) { ErrorSlot error; CapturedOutput output; - const int status = jsoncons_parse_file(parser, path, &error.value); - return {status, error.replaced(), error.value == nullptr, - error.value != nullptr ? error.value : "", output.read()}; + const int status = jsoncons_parse_file(parser, path, error.slot()); + return {status, error.replaced(), error.get() == nullptr, + error.get() != nullptr ? error.get() : "", output.read()}; } -ParseResult parse(jsoncons_parser* parser, const std::string& text) { +ParseResult parse(jsoncons_parser* parser, std::string_view text) { TemporaryInput input(text); return parse_path(parser, input.path()); } -class Checks { - unsigned int total_ = 0; - unsigned int failures_ = 0; - +class Checks : public jsonbench::test::TestChecks { public: - void check(const std::string& name, bool passed) { - ++total_; - if (!passed) { - ++failures_; - std::fprintf(stderr, "FAIL: %s\n", name.c_str()); - } - } - void success(const std::string& name, const ParseResult& result, - const std::string& expected = "") { - check(name + ": succeeds", result.status == 0); - check(name + ": clears previous error", result.error_replaced && result.error_is_null); - check(name + ": exact output", result.output == expected); - if (result.status != 0) - std::fprintf(stderr, " diagnostic: %s\n", result.error.c_str()); - } - void failure(const std::string& name, const ParseResult& result) { - check(name + ": fails", result.status != 0); - check(name + ": returns fresh malloc-owned diagnostic", - result.error_replaced && !result.error_is_null && !result.error.empty()); + void success(std::string_view name, const ParseResult& result, std::string_view expected = "") { + success3(name, result, expected, {": succeeds", ": clears previous error", ": exact output"}); } - int finish() const { - std::printf("jsoncons C API: %u checks passed, %u failed\n", - total_ - failures_, failures_); - return failures_ == 0 ? 0 : 1; + void failure(std::string_view name, const ParseResult& result) { + failure2(name, result, result.status != 0, {": fails", ": returns fresh malloc-owned diagnostic"}); } + int finish() const { return report("jsoncons C API"); } }; void invalid_arguments(Checks& checks) { { ErrorSlot error; - checks.check("init rejects null parser slot", jsoncons_json_init(nullptr, &error.value) != 0); - checks.check("failed init clears previous error", error.replaced() && error.value == nullptr); + checks.check("init rejects null parser slot", jsoncons_json_init(nullptr, error.slot()) != 0); + checks.check("failed init clears previous error", error.replaced() && error.get() == nullptr); } jsoncons_parser* null_parser = nullptr; checks.check("init rejects null error slot", jsoncons_json_init(&null_parser, nullptr) != 0); @@ -197,10 +81,10 @@ void invalid_arguments(Checks& checks) { TemporaryInput input("null"); const ParseResult no_parser = parse_path(nullptr, input.path()); checks.check("parse rejects null parser", no_parser.status != 0); - checks.check("invalid parser clears previous error", no_parser.error_replaced && no_parser.error_is_null); + checks.check("invalid parser clears previous error", no_parser.replaced && no_parser.null_error); const ParseResult no_filename = parse_path(parser.value, nullptr); checks.check("parse rejects null filename", no_filename.status != 0); - checks.check("invalid filename clears previous error", no_filename.error_replaced && no_filename.error_is_null); + checks.check("invalid filename clears previous error", no_filename.replaced && no_filename.null_error); checks.check("parse rejects null error slot", jsoncons_parse_file(parser.value, input.path(), nullptr) != 0); checks.failure("missing file", parse_path(parser.value, (std::string(input.path()) + ".missing").c_str())); @@ -221,37 +105,37 @@ void limit_setters(Checks& checks) { {"fraction below one", 0.5}, {"positive fraction", 1.5}, {"above UINT_MAX", upper + 1.0}, {"DBL_MAX", std::numeric_limits::max()} }; - for (const auto& value : invalid) { - checks.check("depth rejects " + value.first, - jsoncons_set_max_depth(parser.value, value.second) != 0); - checks.check("argument count rejects " + value.first, - jsoncons_set_max_arg_num(parser.value, value.second) != 0); - checks.success("rejected " + value.first + " preserves limits", - parse(parser.value, "{\"a\":{\"b\":null,\"c\":true}}")); - checks.failure("rejected " + value.first + " preserves depth ceiling", + for (const auto& [name, value] : invalid) { + checks.check("depth rejects " + name, + jsoncons_set_max_depth(parser.value, value) != 0); + checks.check("argument count rejects " + name, + jsoncons_set_max_arg_num(parser.value, value) != 0); + checks.success("rejected " + name + " preserves limits", + parse(parser.value, R"({"a":{"b":null,"c":true}})")); + checks.failure("rejected " + name + " preserves depth ceiling", parse(parser.value, "[[[null]]]")); - checks.failure("rejected " + value.first + " preserves count ceiling", + checks.failure("rejected " + name + " preserves count ceiling", parse(parser.value, "[null,true,false]")); } checks.check("depth accepts UINT_MAX", jsoncons_set_max_depth(parser.value, upper) == 0); checks.check("argument count accepts UINT_MAX", jsoncons_set_max_arg_num(parser.value, upper) == 0); checks.success("maximum representable limits permit ordinary JSON", - parse(parser.value, "{\"a\":[null,true,false]}")); + parse(parser.value, R"({"a":[null,true,false]})")); } void zero_limits(Checks& checks) { Parser parser; checks.check("set depth to zero", jsoncons_set_max_depth(parser.value, 0) == 0); checks.check("enable silence for zero depth", jsoncons_set_silence(parser.value, 1) == 0); - for (const auto& text : {"null", "true", "false", "1", "\"value\""}) + for (const auto& text : {"null", "true", "false", "1", R"("value")"}) checks.success(std::string("zero depth allows root ") + text, parse(parser.value, text)); checks.failure("zero depth rejects object", parse(parser.value, "{}")); checks.failure("zero depth rejects array", parse(parser.value, "[]")); checks.check("restore depth for zero arguments", jsoncons_set_max_depth(parser.value, 3) == 0); checks.check("set arguments to zero", jsoncons_set_max_arg_num(parser.value, 0) == 0); - for (const auto& text : {"{}", "[]", "{\"a\":[{},[]]}"}) + for (const auto& text : {"{}", "[]", R"({"a":[{},[]]})"}) checks.success(std::string("zero arguments allows ") + text, parse(parser.value, text)); - for (const auto& text : {"null", "true", "false", "0", "\"\"", "{\"a\":null}", "[false]"}) + for (const auto& text : {"null", "true", "false", "0", R"("")", R"({"a":null})", "[false]"}) checks.failure(std::string("zero arguments rejects ") + text, parse(parser.value, text)); checks.check("negative zero is an integral zero limit", jsoncons_set_max_depth(parser.value, -0.0) == 0); checks.failure("negative-zero depth has zero semantics", parse(parser.value, "{}")); @@ -264,33 +148,33 @@ void boundaries_and_reuse(Checks& checks) { checks.check("silence accepts any nonzero value", jsoncons_set_silence(parser.value, -7) == 0); checks.success("array exactly at depth", parse(parser.value, "[[[null]]]")); checks.failure("array above depth", parse(parser.value, "[[[[null]]]]")); - checks.success("object exactly at depth", parse(parser.value, "{\"a\":{\"b\":{\"c\":true}}}")); - checks.failure("object above depth", parse(parser.value, "{\"a\":{\"b\":{\"c\":{}}}}")); - checks.success("null and bool exactly at count", parse(parser.value, "{\"first\":null,\"second\":true}")); - checks.failure("null and bool above count", parse(parser.value, "{\"first\":null,\"second\":true,\"third\":false}")); - checks.success("empty containers consume no arguments", parse(parser.value, "{\"a\":{},\"b\":[],\"c\":null,\"d\":false}")); - checks.success("success resets count and path", parse(parser.value, "{\"new\":null,\"fresh\":false}")); - const ParseResult malformed = parse(parser.value, "{\"leaked\":{\"a\":true,\"b\":"); + checks.success("object exactly at depth", parse(parser.value, R"({"a":{"b":{"c":true}}})")); + checks.failure("object above depth", parse(parser.value, R"({"a":{"b":{"c":{}}}})")); + checks.success("null and bool exactly at count", parse(parser.value, R"({"first":null,"second":true})")); + checks.failure("null and bool above count", parse(parser.value, R"({"first":null,"second":true,"third":false})")); + checks.success("empty containers consume no arguments", parse(parser.value, R"({"a":{},"b":[],"c":null,"d":false})")); + checks.success("success resets count and path", parse(parser.value, R"({"new":null,"fresh":false})")); + const ParseResult malformed = parse(parser.value, R"({"leaked":{"a":true,"b":)"); checks.failure("nested truncated input", malformed); checks.check("syntax error reports line", malformed.error.find("line ") != std::string::npos); checks.check("syntax error reports column", malformed.error.find("column ") != std::string::npos); checks.check("silent error produces no arguments", malformed.output.empty()); - checks.success("syntax failure resets count and path", parse(parser.value, "{\"fresh\":true}")); + checks.success("syntax failure resets count and path", parse(parser.value, R"({"fresh":true})")); checks.check("disable silence after reused parser", jsoncons_set_silence(parser.value, 0) == 0); - checks.success("output has fresh path after nested failure", parse(parser.value, "{\"fresh\":true}"), "fresh: true\n"); + checks.success("output has fresh path after nested failure", parse(parser.value, R"({"fresh":true})"), "fresh: true\n"); checks.success("successful root array resets afterward", parse(parser.value, "[null,false]"), "array.array: \narray.array: false\n"); checks.success("root scalar has fresh empty path", parse(parser.value, "2"), ": 2\n"); - const ParseResult count_error = parse(parser.value, "{\"a\":null,\"b\":true,\"c\":false}"); + const ParseResult count_error = parse(parser.value, R"({"a":null,"b":true,"c":false})"); checks.failure("retains count limit when unsilenced", count_error); checks.check("count limit prints exactly permitted prefix", count_error.output == "a: \nb: true\n"); - checks.success("count failure resets parser", parse(parser.value, "{\"after\":false}"), "after: false\n"); + checks.success("count failure resets parser", parse(parser.value, R"({"after":false})"), "after: false\n"); checks.check("tighten depth retained across calls", jsoncons_set_max_depth(parser.value, 1) == 0); - checks.failure("retains depth limit across calls", parse(parser.value, "{\"too\":[]}")); - checks.success("depth failure resets parser", parse(parser.value, "{\"after\":null}"), "after: \n"); + checks.failure("retains depth limit across calls", parse(parser.value, R"({"too":[]})")); + checks.success("depth failure resets parser", parse(parser.value, R"({"after":null})"), "after: \n"); checks.check("reenable silence", jsoncons_set_silence(parser.value, 2) == 0); - checks.success("silence persists on successful reuse", parse(parser.value, "{\"quiet\":false}")); - checks.failure("silence still validates malformed JSON", parse(parser.value, "{\"quiet\":true,}")); - checks.success("silence persists after failed reuse", parse(parser.value, "{\"quiet\":true}")); + checks.success("silence persists on successful reuse", parse(parser.value, R"({"quiet":false})")); + checks.failure("silence still validates malformed JSON", parse(parser.value, R"({"quiet":true,})")); + checks.success("silence persists after failed reuse", parse(parser.value, R"({"quiet":true})")); } } // namespace diff --git a/tests/pathhandler_tests.cpp b/tests/pathhandler_tests.cpp index da0d70b..c326334 100644 --- a/tests/pathhandler_tests.cpp +++ b/tests/pathhandler_tests.cpp @@ -1,44 +1,35 @@ // Public path-builder contracts shared by SAX adapters. These tests do not // depend on private stack layout or one particular parser implementation. #include "../src/pathhandler.h" +#include "test_support.h" +#include #include #include -#include +#include #include +#include #include namespace { -class Checks { - unsigned int total_ = 0; - unsigned int failures_ = 0; +class Checks : public jsonbench::test::TestChecks { public: - void check(const std::string& name, bool passed) { - ++total_; - if (!passed) { - ++failures_; - std::fprintf(stderr, "FAIL: %s\n", name.c_str()); - } - } - void path(const std::string& label, PathHandler& handler, const std::string& expected) { - check(label + ": byte-exact name", handler.name() == expected); - check(label + ": no path error", !handler.m_error); + void path(std::string_view label, PathHandler& handler, std::string_view expected) { + check(std::string(label) + ": byte-exact name", handler.name() == expected); + check(std::string(label) + ": no path error", !handler.m_error); // Both NUL terminators and bytes beyond the declared destination size // matter: oversized writes must not corrupt adjacent memory. std::vector buffer(JSON_STRING_SIZE + 16, '#'); handler.buildArgName(buffer.data() + 8); - check(label + ": legacy output bytes", std::memcmp(buffer.data() + 8, + check(std::string(label) + ": legacy output bytes", std::memcmp(buffer.data() + 8, expected.data(), expected.size()) == 0); - check(label + ": legacy output terminator", buffer[8 + expected.size()] == '\0'); + check(std::string(label) + ": legacy output terminator", buffer[8 + expected.size()] == '\0'); bool canaries = true; for (unsigned int i = 0; i < 8; ++i) canaries = canaries && buffer[i] == '#' && buffer[JSON_STRING_SIZE + 8 + i] == '#'; - check(label + ": destination canaries unchanged", canaries); - } - int finish() const { - std::printf("PathHandler: %u checks passed, %u failed\n", total_ - failures_, failures_); - return failures_ ? 1 : 0; + check(std::string(label) + ": destination canaries unchanged", canaries); } + int finish() const { return report("PathHandler"); } }; void structural_paths(Checks& checks) { @@ -111,14 +102,14 @@ void empty_and_binary_keys(Checks& checks) { checks.path("empty root restored", empty, ""); PathHandler binary; - const char parent[] = {'a', '\0', 'b'}; - const char child[] = {'c', '\0', 'd'}; - const std::string parent_string(parent, sizeof parent); - const std::string child_string(child, sizeof child); + constexpr std::array parent = {'a', '\0', 'b'}; + constexpr std::array child = {'c', '\0', 'd'}; + const std::string parent_string(parent.data(), parent.size()); + const std::string child_string(child.data(), child.size()); binary.push(); - binary.setKey(parent, sizeof parent); + binary.setKey(parent.data(), parent.size()); binary.push(); - binary.setKey(child, sizeof child); + binary.setKey(child.data(), child.size()); checks.path("embedded NUL keys preserve bytes", binary, parent_string + "." + child_string); binary.push(); binary.setKey("leaf", 4); @@ -169,7 +160,7 @@ void boundaries_and_errors(Checks& checks) { std::vector untouched(limit, '!'); too_long.buildArgName(untouched.data()); checks.check("errored legacy conversion writes no bytes", - untouched == std::vector(limit, '!')); + untouched == std::vector(limit, '!')); PathHandler push_limit; push_limit.setKey(parent.data(), parent.size()); diff --git a/tests/regression_tests.py b/tests/regression_tests.py index 8ea5dcd..27e1820 100755 --- a/tests/regression_tests.py +++ b/tests/regression_tests.py @@ -362,6 +362,8 @@ def operational(self, engine): for option in ("-d", "-a"): for value in ("-1", "abc", "1x", "1.5", "4294967296", "0"): self.rejected(engine, f"CLI rejects {option} {value}", path, [option, value]) + for value in ("-1", "abc", "1x", "1.5", "2147483648", "4294967297", "0"): + self.rejected(engine, f"CLI rejects -n {value}", path, ["-n", value]) if Path("/dev/full").exists(): for iterations in (1, 2): with open("/dev/full", "wb") as output: diff --git a/tests/test_support.h b/tests/test_support.h new file mode 100644 index 0000000..8fe6511 --- /dev/null +++ b/tests/test_support.h @@ -0,0 +1,281 @@ +#ifndef JSONBENCH_TEST_SUPPORT_H +#define JSONBENCH_TEST_SUPPORT_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace jsonbench::test { + +class TestError final : public std::runtime_error { + public: + using std::runtime_error::runtime_error; +}; + +class FileDescriptor { + int value_ = -1; + public: + explicit FileDescriptor(int value = -1) noexcept : value_(value) {} + ~FileDescriptor() { if (value_ >= 0) ::close(value_); } + FileDescriptor(const FileDescriptor&) = delete; + FileDescriptor& operator=(const FileDescriptor&) = delete; + int get() const noexcept { return value_; } + int release() noexcept { return std::exchange(value_, -1); } + void reset(int value) noexcept { + if (value_ >= 0) ::close(value_); + value_ = value; + } + void close() { + const int value = release(); + if (value >= 0 && ::close(value) != 0) + throw TestError("temporary file close failed"); + } +}; + +struct DirectoryCleanup { + std::string path; + ~DirectoryCleanup() { ::rmdir(path.c_str()); } +}; + +class PrivateWorkspace { + DirectoryCleanup directory_{create()}; + + static std::string create() { + // mkdtemp creates the directory atomically with owner-only access. + // Use the task's working directory, without a public temporary root. + std::string pattern = "./.jsonbench-tests-XXXXXX"; + if (::mkdtemp(pattern.data()) == nullptr) + throw TestError("private test workspace creation failed"); + return pattern; + } + + public: + PrivateWorkspace() { + struct stat attributes{}; + if (::chmod(directory_.path.c_str(), S_IRWXU) != 0 || + ::stat(directory_.path.c_str(), &attributes) != 0 || + !S_ISDIR(attributes.st_mode) || attributes.st_uid != ::geteuid() || + (attributes.st_mode & 0777) != S_IRWXU) + throw TestError("private test workspace permissions failed"); + } + PrivateWorkspace(const PrivateWorkspace&) = delete; + PrivateWorkspace& operator=(const PrivateWorkspace&) = delete; + const std::string& path() const noexcept { return directory_.path; } +}; + +inline const PrivateWorkspace& workspace() { + static const PrivateWorkspace instance; + return instance; +} + +struct FileCleanup { + std::string path; + bool owned = false; + ~FileCleanup() { if (owned) ::unlink(path.c_str()); } +}; + +class TemporaryFile { + FileCleanup file_{workspace().path() + "/fixture-XXXXXX", false}; + FileDescriptor descriptor_; + public: + TemporaryFile() { + descriptor_.reset(::mkstemp(file_.path.data())); + if (descriptor_.get() < 0) throw TestError("temporary file creation failed"); + file_.owned = true; + if (::fchmod(descriptor_.get(), S_IRUSR | S_IWUSR) != 0) + throw TestError("temporary file permissions failed"); + } + TemporaryFile(const TemporaryFile&) = delete; + TemporaryFile& operator=(const TemporaryFile&) = delete; + const char* path() const noexcept { return file_.path.c_str(); } + FileDescriptor& descriptor() noexcept { return descriptor_; } +}; + +class TemporaryInput { + TemporaryFile file_; + public: + explicit TemporaryInput(std::string_view text) { + std::size_t offset = 0; + while (offset < text.size()) { + const auto count = ::write(file_.descriptor().get(), + text.data() + offset, text.size() - offset); + if (count < 0 && errno == EINTR) continue; + if (count <= 0) throw TestError("temporary input write failed"); + offset += static_cast(count); + } + file_.descriptor().close(); + } + TemporaryInput(const TemporaryInput&) = delete; + TemporaryInput& operator=(const TemporaryInput&) = delete; + const char* path() const noexcept { return file_.path(); } +}; + +class CapturedOutput { + using FileCloser = int (*)(FILE*); + TemporaryFile temporary_; + std::unique_ptr file_{nullptr, &std::fclose}; + FileDescriptor saved_; + bool active_ = false; + public: + CapturedOutput() { + if (std::fflush(stdout) != 0) throw TestError("stdout flush failed"); + file_.reset(::fdopen(temporary_.descriptor().get(), "w+b")); + if (!file_) throw TestError("stdout temporary file failed"); + temporary_.descriptor().release(); // fdopen now owns the descriptor. + saved_.reset(::dup(STDOUT_FILENO)); + if (saved_.get() < 0 || ::dup2(::fileno(file_.get()), STDOUT_FILENO) < 0) + throw TestError("stdout capture failed"); + active_ = true; + } + ~CapturedOutput() { + if (active_) { + std::fflush(stdout); + ::dup2(saved_.get(), STDOUT_FILENO); + } + } + CapturedOutput(const CapturedOutput&) = delete; + CapturedOutput& operator=(const CapturedOutput&) = delete; + std::string read() { + if (std::fflush(stdout) != 0 || std::fseek(file_.get(), 0, SEEK_SET) != 0) + throw TestError("captured output rewind failed"); + std::string result; + std::array buffer{}; + std::size_t length; + while ((length = std::fread(buffer.data(), 1, buffer.size(), file_.get())) != 0) + result.append(buffer.data(), length); + if (std::ferror(file_.get())) throw TestError("captured output read failed"); + return result; + } +}; + +class ErrorSlot { + using Diagnostic = std::unique_ptr; + Diagnostic original_{::strdup(""), &std::free}; + Diagnostic returned_{nullptr, &std::free}; + char* value_ = original_.get(); + public: + ErrorSlot() { + if (!original_) throw TestError("diagnostic seed allocation failed"); + } + ~ErrorSlot() { + // Replacing the slot must not release the caller's previous allocation. + // C API results use the same C allocator as the seed and diagnostics. + if (value_ != original_.get()) returned_.reset(value_); + } + ErrorSlot(const ErrorSlot&) = delete; + ErrorSlot& operator=(const ErrorSlot&) = delete; + char** slot() noexcept { return &value_; } + char* get() const noexcept { return value_; } + bool replaced() const noexcept { return value_ != original_.get(); } +}; + +template +class ApiParser { + public: + typename Api::Handle* value = nullptr; + ApiParser() { + ErrorSlot error; + if (Api::init(&value, error.slot()) != 0 || !value || error.get()) { + Api::cleanup(value); + throw TestError(std::string(Api::name()) + " initialization failed"); + } + } + ~ApiParser() { Api::cleanup(value); } + ApiParser(const ApiParser&) = delete; + ApiParser& operator=(const ApiParser&) = delete; +}; + +struct ParseResult { + int status; + bool replaced; + bool null_error; + std::string error; + std::string output; +}; + +class TestChecks { + unsigned int total_ = 0; + unsigned int failures_ = 0; + std::string engine_; + + static void diagnostic(const ParseResult& result) { + if (result.status != 0) + std::fprintf(stderr, " diagnostic: %s\n", result.error.c_str()); + } + + protected: + void success3(std::string_view name, const ParseResult& result, + std::string_view expected, const std::array& labels) { + check(std::string(name) + labels[0], result.status == 0); + check(std::string(name) + labels[1], result.replaced && result.null_error); + check(std::string(name) + labels[2], result.output == expected); + diagnostic(result); + } + void success2(std::string_view name, const ParseResult& result, std::string_view expected) { + check(std::string(name) + ": success and reset diagnostic", + result.status == 0 && result.replaced && result.null_error); + check(std::string(name) + ": exact output", result.output == expected); + diagnostic(result); + } + void failure2(std::string_view name, const ParseResult& result, bool failed, + const std::array& labels) { + check(std::string(name) + labels[0], failed); + check(std::string(name) + labels[1], result.replaced && + !result.null_error && !result.error.empty()); + } + void failure1(std::string_view name, const ParseResult& result, std::string_view expected) { + check(std::string(name) + ": failure and owned diagnostic", result.status != 0 && + result.replaced && !result.null_error && !result.error.empty()); + if (!expected.empty()) + check(std::string(name) + ": diagnostic identifies limit", + result.error.find(expected) != std::string::npos); + } + int report(std::string_view suite) const { + std::printf("%.*s: %u checks passed, %u failed\n", + static_cast(suite.size()), suite.data(), passed(), failures_); + return failures_ ? 1 : 0; + } + + public: + void set_engine(std::string_view engine) { engine_ = engine; } + void check(std::string_view name, bool passed) { + ++total_; + if (passed) return; + ++failures_; + const std::string prefix = engine_.empty() ? "" : engine_ + ": "; + std::fprintf(stderr, "FAIL: %s%.*s\n", prefix.c_str(), + static_cast(name.size()), name.data()); + } + unsigned int total() const noexcept { return total_; } + unsigned int failures() const noexcept { return failures_; } + unsigned int passed() const noexcept { return total_ - failures_; } +}; + +template +ParseResult parse_path(typename Api::Handle* parser, const char* path) { + ErrorSlot error; + CapturedOutput output; + const int status = Api::file(parser, path, error.slot()); + return {status, error.replaced(), !error.get(), + error.get() ? error.get() : "", output.read()}; +} + +template +ParseResult parse_file(typename Api::Handle* parser, std::string_view text) { + TemporaryInput input(text); + return parse_path(parser, input.path()); +} + +} // namespace jsonbench::test + +#endif diff --git a/tests/yajl_api_tests.cpp b/tests/yajl_api_tests.cpp index d24805f..718fe06 100644 --- a/tests/yajl_api_tests.cpp +++ b/tests/yajl_api_tests.cpp @@ -1,115 +1,32 @@ // Public C API regressions, including boundaries unavailable through the CLI. // Compile the YAJL adapter as C, then link this harness with the enabled YAJL. #include "../src/yajlparser.h" +#include "test_support.h" -#include #include -#include -#include -#include -#include +#include #include +#include #include #include -#include - #if HAVE_YAJL namespace { +using namespace std::string_view_literals; -class TemporaryInput { - std::string path_; - public: - explicit TemporaryInput(const std::string& text) { - char name[] = "/tmp/jsonbench-yajl-api-XXXXXX"; - const int fd = mkstemp(name); - if (fd < 0) throw std::runtime_error("mkstemp failed"); - path_ = name; - std::size_t offset = 0; - while (offset < text.size()) { - const ssize_t count = write(fd, text.data() + offset, text.size() - offset); - if (count < 0 && errno == EINTR) continue; - if (count <= 0) { - close(fd); - unlink(path_.c_str()); - throw std::runtime_error("temporary input write failed"); - } - offset += static_cast(count); - } - if (close(fd) != 0) { - unlink(path_.c_str()); - throw std::runtime_error("temporary input close failed"); - } - } - ~TemporaryInput() { unlink(path_.c_str()); } - TemporaryInput(const TemporaryInput&) = delete; - TemporaryInput& operator=(const TemporaryInput&) = delete; - const char* path() const { return path_.c_str(); } -}; - -class CapturedOutput { - FILE* file_ = nullptr; - int saved_fd_ = -1; - public: - CapturedOutput() { - if (std::fflush(stdout) != 0) throw std::runtime_error("stdout flush failed"); - file_ = std::tmpfile(); - if (!file_) throw std::runtime_error("stdout temporary file failed"); - saved_fd_ = dup(STDOUT_FILENO); - if (saved_fd_ < 0 || dup2(fileno(file_), STDOUT_FILENO) < 0) { - if (saved_fd_ >= 0) close(saved_fd_); - std::fclose(file_); - throw std::runtime_error("stdout capture failed"); - } - } - ~CapturedOutput() { - std::fflush(stdout); - dup2(saved_fd_, STDOUT_FILENO); - close(saved_fd_); - std::fclose(file_); - } - CapturedOutput(const CapturedOutput&) = delete; - CapturedOutput& operator=(const CapturedOutput&) = delete; - std::string read() { - if (std::fflush(stdout) != 0 || std::fseek(file_, 0, SEEK_SET) != 0) - throw std::runtime_error("captured output rewind failed"); - std::string result; - char buffer[1024]; - std::size_t length; - while ((length = std::fread(buffer, 1, sizeof buffer, file_)) != 0) - result.append(buffer, length); - if (std::ferror(file_)) throw std::runtime_error("captured output read failed"); - return result; - } -}; - -// Callers own both their old diagnostic and each newly returned diagnostic. -// Starting with an allocation also catches APIs incorrectly freeing old errors. -class ErrorSlot { - char* original_; - public: - char* value; - ErrorSlot() : original_(static_cast(std::malloc(1))), value(original_) { - if (!original_) throw std::bad_alloc(); - *original_ = '\0'; - } - ~ErrorSlot() { - if (value != original_) std::free(value); - std::free(original_); - } - ErrorSlot(const ErrorSlot&) = delete; - ErrorSlot& operator=(const ErrorSlot&) = delete; - bool replaced() const { return value != original_; } -}; +using jsonbench::test::CapturedOutput; +using jsonbench::test::ErrorSlot; +using jsonbench::test::TemporaryInput; +using jsonbench::test::TestError; class Parser { public: yajl_json_data* value = nullptr; Parser() { ErrorSlot error; - if (yajl_json_init(&value, &error.value) != 1 || !value || error.value) { + if (yajl_json_init(&value, error.slot()) != 1 || !value || error.get()) { yajl_json_cleanup(value); - throw std::runtime_error("parser initialization failed"); + throw TestError("parser initialization failed"); } } ~Parser() { yajl_json_cleanup(value); } @@ -117,62 +34,37 @@ class Parser { Parser& operator=(const Parser&) = delete; }; -struct Result { - int status; - bool replaced; - bool null_error; - std::string error; - std::string output; -}; +using Result = jsonbench::test::ParseResult; Result parse_path(yajl_json_data* parser, const char* path) { ErrorSlot error; CapturedOutput output; - const int status = yajl_parse_file(parser, path, &error.value); - return {status, error.replaced(), !error.value, - error.value ? error.value : "", output.read()}; + const int status = yajl_parse_file(parser, path, error.slot()); + return {status, error.replaced(), !error.get(), + error.get() ? error.get() : "", output.read()}; } -Result parse(yajl_json_data* parser, const std::string& text) { +Result parse(yajl_json_data* parser, std::string_view text) { TemporaryInput input(text); return parse_path(parser, input.path()); } -class Checks { - unsigned int total_ = 0; - unsigned int failures_ = 0; +class Checks : public jsonbench::test::TestChecks { public: - void check(const std::string& name, bool passed) { - ++total_; - if (!passed) { - ++failures_; - std::fprintf(stderr, "FAIL: %s\n", name.c_str()); - } - } - void success(const std::string& name, const Result& result, - const std::string& output = "") { - check(name + ": succeeds", result.status == 0); - check(name + ": clears old diagnostic", result.replaced && result.null_error); - check(name + ": byte-exact output", result.output == output); - if (result.status != 0) - std::fprintf(stderr, " diagnostic: %s\n", result.error.c_str()); - } - void failure(const std::string& name, const Result& result) { - check(name + ": fails", result.status == -1); - check(name + ": fresh malloc-owned diagnostic", - result.replaced && !result.null_error && !result.error.empty()); + void success(std::string_view name, const Result& result, std::string_view expected = "") { + success3(name, result, expected, {": succeeds", ": clears old diagnostic", ": byte-exact output"}); } - int finish() const { - std::printf("YAJL C API: %u checks passed, %u failed\n", total_ - failures_, failures_); - return failures_ ? 1 : 0; + void failure(std::string_view name, const Result& result) { + failure2(name, result, result.status == -1, {": fails", ": fresh malloc-owned diagnostic"}); } + int finish() const { return report("YAJL C API"); } }; void invalid_arguments(Checks& checks) { { ErrorSlot error; - checks.check("init rejects null parser slot", yajl_json_init(nullptr, &error.value) == -1); - checks.check("invalid init clears old diagnostic", error.replaced() && !error.value); + checks.check("init rejects null parser slot", yajl_json_init(nullptr, error.slot()) == -1); + checks.check("invalid init clears old diagnostic", error.replaced() && !error.get()); } yajl_json_data* parser_slot = nullptr; checks.check("init rejects null diagnostic slot", yajl_json_init(&parser_slot, nullptr) == -1); @@ -181,46 +73,46 @@ void invalid_arguments(Checks& checks) { checks.check("cleanup accepts null", yajl_json_cleanup(nullptr) == 1); Parser parser; TemporaryInput input("null"); - for (const auto& invalid : std::vector>{ + for (const auto& [name, result] : std::vector>{ {"null parser", parse_path(nullptr, input.path())}, {"null filename", parse_path(parser.value, nullptr)}}) { - checks.check("file parse rejects " + invalid.first, invalid.second.status == -1); - checks.check("file parse clears diagnostic for " + invalid.first, - invalid.second.replaced && invalid.second.null_error); + checks.check("file parse rejects " + name, result.status == -1); + checks.check("file parse clears diagnostic for " + name, + result.replaced && result.null_error); } checks.check("file parse rejects null diagnostic slot", yajl_parse_file(parser.value, input.path(), nullptr) == -1); checks.failure("missing file", parse_path(parser.value, (std::string(input.path()) + ".missing").c_str())); { ErrorSlot error; - checks.check("chunk rejects null parser", yajl_json_process_chunk(nullptr, "{}", 2, &error.value) == -1); - checks.check("null parser chunk clears diagnostic", error.replaced() && !error.value); + checks.check("chunk rejects null parser", yajl_json_process_chunk(nullptr, "{}", 2, error.slot()) == -1); + checks.check("null parser chunk clears diagnostic", error.replaced() && !error.get()); } { ErrorSlot error; - checks.check("chunk rejects null nonempty buffer", yajl_json_process_chunk(parser.value, nullptr, 1, &error.value) == -1); - checks.check("null buffer chunk clears diagnostic", error.replaced() && !error.value); + checks.check("chunk rejects null nonempty buffer", yajl_json_process_chunk(parser.value, nullptr, 1, error.slot()) == -1); + checks.check("null buffer chunk clears diagnostic", error.replaced() && !error.get()); } checks.check("chunk rejects null diagnostic slot", yajl_json_process_chunk(parser.value, "{}", 2, nullptr) == -1); checks.check("process alias rejects null diagnostic slot", yajl_json_process(parser.value, "{}", 2, nullptr) == -1); { ErrorSlot error; - checks.check("complete rejects null parser", yajl_json_complete(nullptr, &error.value) == -1); - checks.check("null parser complete clears diagnostic", error.replaced() && !error.value); + checks.check("complete rejects null parser", yajl_json_complete(nullptr, error.slot()) == -1); + checks.check("null parser complete clears diagnostic", error.replaced() && !error.get()); } checks.check("complete rejects null diagnostic slot", yajl_json_complete(parser.value, nullptr) == -1); { ErrorSlot error; - checks.check("empty null buffer is a valid chunk", yajl_json_process_chunk(parser.value, nullptr, 0, &error.value) == 1); - checks.check("empty chunk clears diagnostic", error.replaced() && !error.value); + checks.check("empty null buffer is a valid chunk", yajl_json_process_chunk(parser.value, nullptr, 0, error.slot()) == 1); + checks.check("empty chunk clears diagnostic", error.replaced() && !error.get()); } checks.success("parser usable after invalid calls", parse(parser.value, "null"), ": \n"); } -void split_case(Checks& checks, const std::string& name, const std::string& input, - bool valid, const std::string& expected = "") { +void split_case(Checks& checks, std::string_view name, std::string_view input, + bool valid, std::string_view expected = "") { for (std::size_t split = 0; split <= input.size(); ++split) { - const std::string label = name + " split " + std::to_string(split); + const std::string label = std::string(name) + " split " + std::to_string(split); Parser parser; parser.value->silence = valid ? 0 : 1; CapturedOutput output; @@ -231,36 +123,36 @@ void split_case(Checks& checks, const std::string& name, const std::string& inpu ErrorSlot error; // Exercise both public entry points with identical semantics. const int rc = part ? yajl_json_process(parser.value, input.data() + offset, - static_cast(length), &error.value) : + static_cast(length), error.slot()) : yajl_json_process_chunk(parser.value, input.data() + offset, - static_cast(length), &error.value); - checks.check(label + ": chunk replaces old diagnostic", error.replaced()); - checks.check(label + ": chunk result and diagnostic agree", - (rc == 1 && !error.value) || (rc == -1 && error.value && *error.value)); + static_cast(length), error.slot()); + checks.check(std::string(label) + ": chunk replaces old diagnostic", error.replaced()); + checks.check(std::string(label) + ": chunk result and diagnostic agree", + (rc == 1 && !error.get()) || (rc == -1 && error.get() && *error.get())); failed = rc != 1; } if (!failed) { ErrorSlot error; - const int rc = yajl_json_complete(parser.value, &error.value); - checks.check(label + ": complete replaces old diagnostic", error.replaced()); - checks.check(label + ": complete result and diagnostic agree", - (rc == 1 && !error.value) || (rc == -1 && error.value && *error.value)); + const int rc = yajl_json_complete(parser.value, error.slot()); + checks.check(std::string(label) + ": complete replaces old diagnostic", error.replaced()); + checks.check(std::string(label) + ": complete result and diagnostic agree", + (rc == 1 && !error.get()) || (rc == -1 && error.get() && *error.get())); failed = rc != 1; } - checks.check(label + ": expected acceptance", failed != valid); - checks.check(label + ": byte-exact output", output.read() == (valid ? expected : "")); + checks.check(std::string(label) + ": expected acceptance", failed != valid); + checks.check(std::string(label) + ": byte-exact output", output.read() == (valid ? expected : "")); if (valid) { ErrorSlot complete_error; - checks.check(label + ": rejects duplicate completion", - yajl_json_complete(parser.value, &complete_error.value) == -1); - checks.check(label + ": duplicate completion has fresh diagnostic", - complete_error.replaced() && complete_error.value && *complete_error.value); + checks.check(std::string(label) + ": rejects duplicate completion", + yajl_json_complete(parser.value, complete_error.slot()) == -1); + checks.check(std::string(label) + ": duplicate completion has fresh diagnostic", + complete_error.replaced() && complete_error.get() && *complete_error.get()); ErrorSlot chunk_error; - checks.check(label + ": rejects chunks after completion", - yajl_json_process_chunk(parser.value, " ", 1, &chunk_error.value) == -1); - checks.check(label + ": post-completion chunk has fresh diagnostic", - chunk_error.replaced() && chunk_error.value && *chunk_error.value); - checks.check(label + ": failed repeated calls do not emit more output", output.read() == expected); + checks.check(std::string(label) + ": rejects chunks after completion", + yajl_json_process_chunk(parser.value, " ", 1, chunk_error.slot()) == -1); + checks.check(std::string(label) + ": post-completion chunk has fresh diagnostic", + chunk_error.replaced() && chunk_error.get() && *chunk_error.get()); + checks.check(std::string(label) + ": failed repeated calls do not emit more output", output.read() == expected); } } } @@ -272,29 +164,29 @@ void chunk_boundaries(Checks& checks) { split_case(checks, "root exponent EOF", "-0.1200e+123", true, ": -0.1200e+123\n"); split_case(checks, "raw multibyte UTF-8", "{\"\xc3\xa4\":\"\xe6\x97\xa5\xf0\x9f\x98\x80\"}", true, "\xc3\xa4: \xe6\x97\xa5\xf0\x9f\x98\x80\n"); - split_case(checks, "surrogate pair", "{\"emoji\":\"\\uD83D\\uDE00\"}", true, "emoji: \xf0\x9f\x98\x80\n"); - split_case(checks, "escaped quote and slash", "{\"quote\":\"a\\\"b\\\\c\"}", true, "quote: a\"b\\c\n"); - const char nul_output[] = "a\0b: x\0y\n"; - split_case(checks, "decoded NUL", "{\"a\\u0000b\":\"x\\u0000y\"}", true, - std::string(nul_output, sizeof nul_output - 1)); - split_case(checks, "root and nested arrays", "[[null,true],{\"k\":false},2]", true, + split_case(checks, "surrogate pair", R"({"emoji":"\uD83D\uDE00"})", true, "emoji: \xf0\x9f\x98\x80\n"); + split_case(checks, "escaped quote and slash", R"({"quote":"a\"b\\c"})", true, "quote: a\"b\\c\n"); + constexpr auto nul_output = "a\0b: x\0y\n"sv; + split_case(checks, "decoded NUL", R"({"a\u0000b":"x\u0000y"})", true, + std::string(nul_output)); + split_case(checks, "root and nested arrays", R"([[null,true],{"k":false},2])", true, "array.array.array: \narray.array.array: true\narray.array.k: false\narray.array: 2\n"); - split_case(checks, "dotted parents and siblings", "{\"a.b\":[{\"c.d\":1},2],\"after\":3}", true, + split_case(checks, "dotted parents and siblings", R"({"a.b":[{"c.d":1},2],"after":3})", true, "a.b.a.b.c.d: 1\na.b.a.b: 2\nafter: 3\n"); - split_case(checks, "repeated and empty keys", "{\"\":{\"\":1},\"k\":2,\"k\":3}", true, + split_case(checks, "repeated and empty keys", R"({"":{"":1},"k":2,"k":3})", true, ": 1\nk: 2\nk: 3\n"); const std::vector> invalid = { {"empty input", ""}, {"whitespace only", " \r\n\t"}, - {"truncated object", "{\"a\":1"}, {"truncated nested array", "[[true]"}, - {"truncated string", "\"text"}, {"truncated number", "1e+"}, + {"truncated object", R"({"a":1)"}, {"truncated nested array", "[[true]"}, + {"truncated string", R"("text)"}, {"truncated number", "1e+"}, {"trailing document", "{}[]"}, {"trailing garbage", "null x"}, {"trailing comma", "[1,]"}, {"missing separator", "[1 2]"}, - {"invalid escape", "\"\\q\""}, {"high surrogate alone", "\"\\ud800\""}, - {"low surrogate alone", "\"\\udc00\""}, - {"high surrogate plus ordinary unit", "\"\\ud800\\u0041\""}, - {"high surrogate plus high unit", "\"\\ud800\\ud800\""}, - {"surrogate plus raw continuation", "\"\\ud800a\""}, + {"invalid escape", R"("\q")"}, {"high surrogate alone", R"("\ud800")"}, + {"low surrogate alone", R"("\udc00")"}, + {"high surrogate plus ordinary unit", R"("\ud800\u0041")"}, + {"high surrogate plus high unit", R"("\ud800\ud800")"}, + {"surrogate plus raw continuation", R"("\ud800a")"}, {"overlong UTF-8", std::string("\"\xc0\xaf\"", 4)}, {"UTF-8 surrogate code point", std::string("\"\xed\xa0\x80\"", 5)}, {"UTF-8 beyond U+10FFFF", std::string("\"\xf4\x90\x80\x80\"", 6)}, @@ -303,7 +195,7 @@ void chunk_boundaries(Checks& checks) { {"raw NUL in string", std::string("\"a\0b\"", 5)}, {"raw NUL after document", std::string("{}\0", 3)} }; - for (const auto& test : invalid) split_case(checks, test.first, test.second, false); + for (const auto& [name, input] : invalid) split_case(checks, name, input, false); // Feed one byte per call as well as two chunks: lexer/guard state must span // several calls, rather than merely preserving a final partial token. @@ -313,12 +205,12 @@ void chunk_boundaries(Checks& checks) { for (std::size_t i = 0; i < input.size(); ++i) { ErrorSlot error; checks.check("byte-at-a-time chunk " + std::to_string(i), - yajl_json_process_chunk(parser.value, input.data() + i, 1, &error.value) == 1 && - error.replaced() && !error.value); + yajl_json_process_chunk(parser.value, input.data() + i, 1, error.slot()) == 1 && + error.replaced() && !error.get()); } ErrorSlot error; - checks.check("byte-at-a-time completion", yajl_json_complete(parser.value, &error.value) == 1 && - error.replaced() && !error.value); + checks.check("byte-at-a-time completion", yajl_json_complete(parser.value, error.slot()) == 1 && + error.replaced() && !error.get()); checks.check("byte-at-a-time byte-exact output", output.read() == "\xc3\xa4: \xf0\x9f\x98\x80\nn: -0e+3\n"); } @@ -326,63 +218,63 @@ void limits_and_reuse(Checks& checks) { Parser parser; parser.value->silence = 1; parser.value->depth_limit = 0; - for (const auto& value : {"null", "true", "false", "1", "\"x\""}) + for (const auto& value : {"null", "true", "false", "1", R"("x")"}) checks.success(std::string("zero depth allows primitive ") + value, parse(parser.value, value)); for (const auto& value : {"{}", "[]"}) checks.failure(std::string("zero depth rejects container ") + value, parse(parser.value, value)); parser.value->depth_limit = 3; parser.value->arg_num_limit = 0; - for (const auto& value : {"{}", "[]", "{\"a\":[{},[]]}"}) + for (const auto& value : {"{}", "[]", R"({"a":[{},[]]})"}) checks.success(std::string("zero arguments allows empty containers ") + value, parse(parser.value, value)); - for (const auto& value : {"null", "true", "false", "0", "\"\"", "{\"a\":null}", "[false]"}) + for (const auto& value : {"null", "true", "false", "0", R"("")", R"({"a":null})", "[false]"}) checks.failure(std::string("zero arguments rejects value ") + value, parse(parser.value, value)); parser.value->arg_num_limit = 2; checks.success("array exactly at depth limit", parse(parser.value, "[[[null]]]")); checks.failure("array above depth limit", parse(parser.value, "[[[[null]]]]")); - checks.success("object exactly at depth limit", parse(parser.value, "{\"a\":{\"b\":{\"c\":true}}}")); - checks.failure("object above depth limit", parse(parser.value, "{\"a\":{\"b\":{\"c\":{}}}}")); - checks.success("bool and null exactly at argument limit", parse(parser.value, "{\"a\":null,\"b\":false}")); + checks.success("object exactly at depth limit", parse(parser.value, R"({"a":{"b":{"c":true}}})")); + checks.failure("object above depth limit", parse(parser.value, R"({"a":{"b":{"c":{}}}})")); + checks.success("bool and null exactly at argument limit", parse(parser.value, R"({"a":null,"b":false})")); checks.failure("bool and null above argument limit", parse(parser.value, "[null,true,false]")); - checks.success("empty containers consume no arguments", parse(parser.value, "{\"a\":{},\"b\":[],\"c\":null,\"d\":true}")); - for (const auto& value : {"null", "true", "false", "0", "\"\""}) { + checks.success("empty containers consume no arguments", parse(parser.value, R"({"a":{},"b":[],"c":null,"d":true})")); + for (const auto& value : {"null", "true", "false", "0", R"("")"}) { checks.success(std::string("repeated scalar exactly at count: ") + value, parse(parser.value, std::string("[") + value + "," + value + "]")); checks.failure(std::string("repeated scalar above count: ") + value, parse(parser.value, std::string("[") + value + "," + value + "," + value + "]")); } - const Result incomplete = parse(parser.value, "{\"leaked\":{\"x\":true,\"y\":"); + const Result incomplete = parse(parser.value, R"({"leaked":{"x":true,"y":)"); checks.failure("nested truncated input", incomplete); checks.check("silent malformed document has no argument output", incomplete.output.empty()); - checks.success("reuse after syntax failure", parse(parser.value, "{\"fresh\":true}")); + checks.success("reuse after syntax failure", parse(parser.value, R"({"fresh":true})")); checks.check("reuse preserves caller settings", parser.value->depth_limit == 3 && parser.value->arg_num_limit == 2 && parser.value->silence == 1); parser.value->silence = 0; - checks.success("fresh path after failed file", parse(parser.value, "{\"fresh\":true}"), "fresh: true\n"); + checks.success("fresh path after failed file", parse(parser.value, R"({"fresh":true})"), "fresh: true\n"); checks.success("root array path", parse(parser.value, "[null,false]"), "array.array: \narray.array: false\n"); checks.success("root scalar clears preceding array path", parse(parser.value, "2"), ": 2\n"); - const Result count_failure = parse(parser.value, "{\"a\":null,\"b\":true,\"c\":false}"); + const Result count_failure = parse(parser.value, R"({"a":null,"b":true,"c":false})"); checks.failure("retained count limit while unsilenced", count_failure); checks.check("count failure emits only permitted arguments", count_failure.output == "a: \nb: true\n"); - checks.success("reuse after count failure", parse(parser.value, "{\"after\":false}"), "after: false\n"); + checks.success("reuse after count failure", parse(parser.value, R"({"after":false})"), "after: false\n"); parser.value->depth_limit = 1; - checks.failure("retained tight depth limit", parse(parser.value, "{\"a\":[]}")); - checks.success("reuse after depth failure", parse(parser.value, "{\"after\":null}"), "after: \n"); + checks.failure("retained tight depth limit", parse(parser.value, R"({"a":[]})")); + checks.success("reuse after depth failure", parse(parser.value, R"({"after":null})"), "after: \n"); checks.failure("file reader finalizes root numeric syntax", parse(parser.value, "1e+")); checks.success("reuse after EOF syntax error", parse(parser.value, "-0.0"), ": -0.0\n"); checks.failure("file reader rejects trailing content", parse(parser.value, "{} true")); checks.success("reuse after trailing content", parse(parser.value, "{}")); parser.value->silence = 5; - checks.success("nonzero silence persists", parse(parser.value, "{\"quiet\":true}")); - checks.failure("silent mode still validates", parse(parser.value, "{\"quiet\":true,}")); - checks.success("silence persists after error", parse(parser.value, "{\"quiet\":false}")); + checks.success("nonzero silence persists", parse(parser.value, R"({"quiet":true})")); + checks.failure("silent mode still validates", parse(parser.value, R"({"quiet":true,})")); + checks.success("silence persists after error", parse(parser.value, R"({"quiet":false})")); // A file parse starts a fresh document even if callers previously used the // incremental API and left a partially consumed or failed document behind. { ErrorSlot error; checks.check("incremental parse rejected after finalized file", - yajl_json_process_chunk(parser.value, "{}", 2, &error.value) == -1 && - error.replaced() && error.value && *error.value); + yajl_json_process_chunk(parser.value, "{}", 2, error.slot()) == -1 && + error.replaced() && error.get() && *error.get()); } checks.success("file parse resets finalized incremental state", parse(parser.value, "{}")); { @@ -390,17 +282,17 @@ void limits_and_reuse(Checks& checks) { incremental.value->silence = 1; incremental.value->depth_limit = 2; incremental.value->arg_num_limit = 1; - const char unfinished[] = "{\"stale\":["; + constexpr auto unfinished = R"({"stale":[)"sv; ErrorSlot chunk_error; checks.check("unfinished incremental document initially accepted", - yajl_json_process_chunk(incremental.value, unfinished, sizeof unfinished - 1, - &chunk_error.value) == 1 && chunk_error.replaced() && !chunk_error.value); + yajl_json_process_chunk(incremental.value, unfinished.data(), static_cast(unfinished.size()), + chunk_error.slot()) == 1 && chunk_error.replaced() && !chunk_error.get()); ErrorSlot complete_error; checks.check("unfinished incremental document rejected on completion", - yajl_json_complete(incremental.value, &complete_error.value) == -1 && - complete_error.replaced() && complete_error.value && *complete_error.value); + yajl_json_complete(incremental.value, complete_error.slot()) == -1 && + complete_error.replaced() && complete_error.get() && *complete_error.get()); checks.success("file parse resets failed incremental document", - parse(incremental.value, "{\"fresh\":true}")); + parse(incremental.value, R"({"fresh":true})")); checks.check("incremental-to-file reset preserves settings", incremental.value->silence == 1 && incremental.value->depth_limit == 2 && incremental.value->arg_num_limit == 1); @@ -414,47 +306,47 @@ void length_boundaries(Checks& checks) { const std::size_t limit = JSON_STRING_SIZE; const std::string key(limit - 1, 'k'); const std::string value(limit - 1, 'v'); - checks.success("key exactly below byte limit", parse(parser.value, "{\"" + key + "\":null}"), key + ": \n"); - checks.failure("key at byte limit", parse(parser.value, "{\"" + key + "k\":null}")); - checks.success("value exactly below byte limit", parse(parser.value, "{\"v\":\"" + value + "\"}"), "v: " + value + "\n"); - checks.failure("value at byte limit", parse(parser.value, "{\"v\":\"" + value + "v\"}")); + checks.success("key exactly below byte limit", parse(parser.value, R"({")" + key + R"(":null})"), key + ": \n"); + checks.failure("key at byte limit", parse(parser.value, R"({")" + key + R"(k":null})")); + checks.success("value exactly below byte limit", parse(parser.value, R"({"v":")" + value + R"("})"), "v: " + value + "\n"); + checks.failure("value at byte limit", parse(parser.value, R"({"v":")" + value + R"(v"})")); const std::string parent((limit - 2) / 2, 'p'); const std::string child(limit - 2 - parent.size(), 'c'); checks.success("full path exactly below byte limit", - parse(parser.value, "{\"" + parent + "\":{\"" + child + "\":1}}"), parent + "." + child + ": 1\n"); + parse(parser.value, R"({")" + parent + R"(":{")" + child + R"(":1}})"), parent + "." + child + ": 1\n"); checks.failure("full path at byte limit", parse(parser.value, - "{\"" + parent + "\":{\"" + child + "c\":1}}")); + R"({")" + parent + R"(":{")" + child + R"(c":1}})")); checks.failure("oversized path entering empty container", parse(parser.value, - "{\"" + parent + "\":{\"" + child + "c\":{}}}")); + R"({")" + parent + R"(":{")" + child + R"(c":{}}})")); // A valid decoded token can span several file-reader chunks. Length limits // apply to decoded bytes, not to the longer escape spelling in the input. std::string escaped_key; std::string escaped_value; for (std::size_t i = 0; i < limit - 1; ++i) { - escaped_key += "\\u006b"; - escaped_value += "\\u0076"; + escaped_key += R"(\u006b)"; + escaped_value += R"(\u0076)"; } checks.success("escaped key below decoded byte limit across file chunks", - parse(parser.value, "{\"" + escaped_key + "\":null}"), key + ": \n"); + parse(parser.value, R"({")" + escaped_key + R"(":null})"), key + ": \n"); checks.failure("escaped key at decoded byte limit", - parse(parser.value, "{\"" + escaped_key + "\\u006b\":null}")); + parse(parser.value, R"({")" + escaped_key + R"(\u006b":null})")); checks.success("escaped value below decoded byte limit across file chunks", - parse(parser.value, "{\"v\":\"" + escaped_value + "\"}"), "v: " + value + "\n"); + parse(parser.value, R"({"v":")" + escaped_value + R"("})"), "v: " + value + "\n"); checks.failure("escaped value at decoded byte limit", - parse(parser.value, "{\"v\":\"" + escaped_value + "\\u0076\"}")); + parse(parser.value, R"({"v":")" + escaped_value + R"(\u0076"})")); parser.value->silence = 1; - checks.failure("silent key length still enforced", parse(parser.value, "{\"" + key + "k\":0}")); - checks.failure("silent value length still enforced", parse(parser.value, "\"" + value + "v\"")); - checks.failure("oversized token guard", parse(parser.value, "\"" + std::string(limit * 8 + 32, 'x') + "\"")); + checks.failure("silent key length still enforced", parse(parser.value, R"({")" + key + R"(k":0})")); + checks.failure("silent value length still enforced", parse(parser.value, R"(")" + value + R"(v")")); + checks.failure("oversized token guard", parse(parser.value, R"(")" + std::string(limit * 8 + 32, 'x') + R"(")")); checks.failure("oversized number guard", parse(parser.value, std::string(limit * 8 + 32, '1'))); - checks.success("reuse after length errors", parse(parser.value, "{\"fresh\":true}")); + checks.success("reuse after length errors", parse(parser.value, R"({"fresh":true})")); parser.value->silence = 0; - const char expected[] = "a\0b.c\0d: v\0x\na\0b.after: false\nlast: \n"; + constexpr auto expected = "a\0b.c\0d: v\0x\na\0b.after: false\nlast: \n"sv; checks.success("NUL parents restore with explicit lengths", parse(parser.value, - "{\"a\\u0000b\":{\"c\\u0000d\":\"v\\u0000x\",\"after\":false},\"last\":null}"), - std::string(expected, sizeof expected - 1)); + R"({"a\u0000b":{"c\u0000d":"v\u0000x","after":false},"last":null})"), + std::string(expected)); checks.success("dotted keys are not structural pop delimiters", parse(parser.value, - "{\"a.b\":{\"c.d\":1},\"after\":2}"), "a.b.c.d: 1\nafter: 2\n"); + R"({"a.b":{"c.d":1},"after":2})"), "a.b.c.d: 1\nafter: 2\n"); } } // namespace From cdc69d165b04377faab042692a0da78360d4b034 Mon Sep 17 00:00:00 2001 From: jens Steinbacher Date: Tue, 13 Oct 2026 17:41:27 +0000 Subject: [PATCH 08/18] Resolve remaining shared Sonar reliability and complexity findings --- src/jsonbench.c | 17 ++-- src/jsonconsparser.cpp | 79 +++++++++++------- src/yajlparser.c | 158 +++++++++++++++++------------------ src/yajlparser.h | 36 ++++---- tests/jsoncons_api_tests.cpp | 2 +- tests/pathhandler_tests.cpp | 2 +- tests/test_support.h | 26 ++++-- tests/yajl_api_tests.cpp | 45 ++++++---- 8 files changed, 204 insertions(+), 161 deletions(-) diff --git a/src/jsonbench.c b/src/jsonbench.c index 6f1317b..5c91544 100644 --- a/src/jsonbench.c +++ b/src/jsonbench.c @@ -258,9 +258,6 @@ static int accept_option(int option, const char *argument, const benchmark_engine **engine) { unsigned int iterations = 0; switch (option) { - case 'h': - showhelp(); - return 1; case 'e': *engine = find_engine(argument); if (*engine != NULL) return 0; @@ -279,6 +276,7 @@ static int accept_option(int option, const char *argument, case 's': settings->silence = 1; return 0; + case 'h': default: showhelp(); return 1; @@ -335,16 +333,17 @@ static int parse_once(const benchmark_engine *engine, } static int benchmark(const benchmark_engine *engine, - const benchmark_settings *settings, double *times) { + const benchmark_settings *settings, double *times, + size_t count) { if (parse_once(engine, settings, 1, 1, NULL) != 0) return EXIT_FAILURE; int timed_silence = settings->silence; - if (settings->iterations > 1) timed_silence = 1; - for (int iteration = 0; iteration < settings->iterations; ++iteration) { + if (count > 1) timed_silence = 1; + for (size_t iteration = 0; iteration < count; ++iteration) { if (parse_once(engine, settings, timed_silence, 0, ×[iteration]) != 0) return EXIT_FAILURE; } - print_stats(times, settings->iterations); - if (settings->iterations > 1 && !settings->silence) { + print_stats(times, (int)count); + if (count > 1 && !settings->silence) { double elapsed; if (parse_once(engine, settings, 0, 0, &elapsed) != 0) return EXIT_FAILURE; } @@ -371,7 +370,7 @@ int main(int argc, char **argv) { fprintf(stderr, "Memory allocation error\n"); return EXIT_FAILURE; } - const int result = benchmark(engine, &settings, times); + const int result = benchmark(engine, &settings, times, count); free(times); return result; } diff --git a/src/jsonconsparser.cpp b/src/jsonconsparser.cpp index 00436f0..00e62aa 100644 --- a/src/jsonconsparser.cpp +++ b/src/jsonconsparser.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -53,6 +54,35 @@ class UnicodeSource { std::to_string(line_) + ", column " + std::to_string(column_)); } + unsigned int hex_digit(unsigned char c) const { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + invalid(); + } + + void complete_codepoint() { + if (low_required_) { + if (codepoint_ < 0xdc00 || codepoint_ > 0xdfff) invalid(); + low_required_ = false; + state_ = State::text; + return; + } + if (codepoint_ >= 0xd800 && codepoint_ <= 0xdbff) { + low_required_ = true; + state_ = State::pair_slash; + return; + } + if (codepoint_ >= 0xdc00 && codepoint_ <= 0xdfff) invalid(); + state_ = State::text; + } + + void start_hex() { + state_ = State::hex; + codepoint_ = 0; + digits_ = 0; + } + void check(unsigned char c) { if (c == '\n') { ++line_; column_ = 0; } else { ++column_; } switch (state_) { @@ -65,32 +95,14 @@ class UnicodeSource { break; case State::escape: if (c == 'u') { - state_ = State::hex; - codepoint_ = 0; - digits_ = 0; + start_hex(); } else { state_ = State::text; } break; case State::hex: { - unsigned int digit; - if (c >= '0' && c <= '9') digit = c - '0'; - else if (c >= 'a' && c <= 'f') digit = c - 'a' + 10; - else if (c >= 'A' && c <= 'F') digit = c - 'A' + 10; - else invalid(); - codepoint_ = codepoint_ * 16 + digit; - if (++digits_ != 4) break; - if (low_required_) { - if (codepoint_ < 0xdc00 || codepoint_ > 0xdfff) invalid(); - low_required_ = false; - state_ = State::text; - } else if (codepoint_ >= 0xd800 && codepoint_ <= 0xdbff) { - low_required_ = true; - state_ = State::pair_slash; - } else { - if (codepoint_ >= 0xdc00 && codepoint_ <= 0xdfff) invalid(); - state_ = State::text; - } + codepoint_ = codepoint_ * 16 + hex_digit(c); + if (++digits_ == 4) complete_codepoint(); break; } case State::pair_slash: @@ -99,16 +111,16 @@ class UnicodeSource { break; case State::pair_u: if (c != 'u') invalid(); - state_ = State::hex; - codepoint_ = 0; - digits_ = 0; + start_hex(); break; } } public: using value_type = char; - explicit UnicodeSource(std::istream& input) : source_(input) {} + // Keep the stream mutable: jsoncons consumes its buffer and changes its + // EOF/error flags. A reference wrapper explicitly retains that borrowing. + explicit UnicodeSource(std::reference_wrapper input) : source_(input.get()) {} UnicodeSource(UnicodeSource&&) = default; bool eof() const { return source_.eof(); } bool is_error() const { return source_.is_error(); } @@ -124,11 +136,14 @@ class UnicodeSource { // multi-megabyte string/number cannot consume document-sized memory first. // Leave room for geometric growth and the parser's structural state stack. template -class TemporaryAllocator { +class TemporaryAllocator : public std::allocator { public: using value_type = T; TemporaryAllocator() = default; - template TemporaryAllocator(const TemporaryAllocator&) {} + // Reuse the standard allocator's conversion constructors. jsoncons needs + // implicit allocator rebinding when it constructs the parser state vector. + using std::allocator::allocator; + template struct rebind { using other = TemporaryAllocator; }; T* allocate(std::size_t count) { constexpr std::size_t budget = static_cast(JSON_STRING_SIZE) * 4 + 16384; @@ -139,8 +154,10 @@ class TemporaryAllocator { void deallocate(T* data, std::size_t count) noexcept { std::allocator{}.deallocate(data, count); } - template bool operator==(const TemporaryAllocator&) const { return true; } - template bool operator!=(const TemporaryAllocator&) const { return false; } + template + friend bool operator==(const TemporaryAllocator&, const TemporaryAllocator&) noexcept { return true; } + template + friend bool operator!=(const TemporaryAllocator&, const TemporaryAllocator&) noexcept { return false; } }; // Keep structural boundaries explicitly so dots, empty keys and NULs remain @@ -297,7 +314,7 @@ class ArgumentVisitor final : public jsoncons::default_json_visitor { const char* error() const { return error_; } }; -int set_limit(jsoncons_parser* parser, double value, unsigned int* target) { +int set_limit(const jsoncons_parser* parser, double value, unsigned int* target) { if (parser == nullptr || !std::isfinite(value) || value < 0 || value > std::numeric_limits::max() || std::floor(value) != value) { return -1; @@ -348,7 +365,7 @@ extern "C" int jsoncons_parse_file(jsoncons_parser* parser, const char* filename parser->max_depth > static_cast(INT_MAX) ? INT_MAX : parser->max_depth)); jsoncons::basic_json_reader> reader(input, visitor, options); + TemporaryAllocator> reader(std::reference_wrapper(input), visitor, options); std::error_code ec; reader.read(ec); // Includes finalization and checks trailing content. if (ec) { diff --git a/src/yajlparser.c b/src/yajlparser.c index 59ee5bf..3712bd9 100644 --- a/src/yajlparser.c +++ b/src/yajlparser.c @@ -14,7 +14,7 @@ struct yajl_path_frame { }; static int reject(yajl_json_data *p, const char *message) { - p->callback_error = message; + p->state.callback_error = message; return 0; } @@ -64,25 +64,25 @@ static int start_container(yajl_json_data *p, int array) { p->depth_limit_exceeded = 1; return reject(p, "JSON depth limit exceeded"); } - int root_array = array && p->frame_count == 0; + int root_array = array && p->state.frame_count == 0; size_t length = 5; if (!root_array) length = p->prefix_len + (p->prefix_len ? 1 : 0) + p->current_key_len; if (length >= JSON_STRING_SIZE) return reject(p, "Argument name too long"); - if (p->frame_count == p->frame_capacity) { - if (p->frame_capacity > SIZE_MAX / (2 * sizeof(yajl_path_frame))) + if (p->state.frame_count == p->state.frame_capacity) { + if (p->state.frame_capacity > SIZE_MAX / (2 * sizeof(yajl_path_frame))) return reject(p, "JSON path allocation limit exceeded"); - size_t capacity = p->frame_capacity ? p->frame_capacity * 2 : 16; - yajl_path_frame *frames = realloc(p->frames, capacity * sizeof(*frames)); + size_t capacity = p->state.frame_capacity ? p->state.frame_capacity * 2 : 16; + yajl_path_frame *frames = realloc(p->state.frames, capacity * sizeof(*frames)); if (!frames) return reject(p, "Unable to allocate JSON path stack"); - p->frames = frames; - p->frame_capacity = capacity; + p->state.frames = frames; + p->state.frame_capacity = capacity; } unsigned char *saved = malloc(p->current_key_len + 1); if (!saved) return reject(p, "Unable to allocate JSON path key"); memcpy(saved, p->current_key, p->current_key_len); saved[p->current_key_len] = '\0'; - p->frames[p->frame_count++] = (yajl_path_frame){p->prefix_len, p->current_key_len, saved}; + p->state.frames[p->state.frame_count++] = (yajl_path_frame){p->prefix_len, p->current_key_len, saved}; if (root_array) { memcpy(p->prefix, "array", 5); memcpy(p->current_key, "array", 5); @@ -102,8 +102,8 @@ static int start_object(void *ctx) { return start_container(ctx, 0); } static int start_array(void *ctx) { return start_container(ctx, 1); } static int end_container(void *ctx) { yajl_json_data *p = ctx; - if (!p->frame_count) return reject(p, "Unpaired JSON container event"); - yajl_path_frame *frame = &p->frames[--p->frame_count]; + if (!p->state.frame_count) return reject(p, "Unpaired JSON container event"); + yajl_path_frame *frame = &p->state.frames[--p->state.frame_count]; p->prefix_len = frame->prefix_length; p->prefix[p->prefix_len] = '\0'; p->current_key_len = frame->key_length; @@ -118,8 +118,8 @@ static const yajl_callbacks callbacks = { }; static void clear_state(yajl_json_data *p) { - for (size_t i = 0; i < p->frame_count; ++i) free(p->frames[i].key); - p->frame_count = 0; + for (size_t i = 0; i < p->state.frame_count; ++i) free(p->state.frames[i].key); + p->state.frame_count = 0; p->prefix_len = 0; p->current_key_len = 0; p->prefix[0] = '\0'; @@ -128,17 +128,17 @@ static void clear_state(yajl_json_data *p) { p->current_arg_num = 0; p->depth_limit_exceeded = 0; p->arg_num_limit_exceeded = 0; - p->callback_error = NULL; - p->escape_state = 0; - p->escape_digits = 0; - p->escape_codepoint = 0; - p->low_surrogate_required = 0; - p->number_token = 0; - p->completed = 0; - p->token_length = 0; - p->utf8_remaining = 0; - p->utf8_min = 0x80; - p->utf8_max = 0xbf; + p->state.callback_error = NULL; + p->state.escape_state = 0; + p->state.escape_digits = 0; + p->state.escape_codepoint = 0; + p->state.low_surrogate_required = 0; + p->state.number_token = 0; + p->state.completed = 0; + p->state.token_length = 0; + p->state.utf8_remaining = 0; + p->state.utf8_min = 0x80; + p->state.utf8_max = 0xbf; p->status = yajl_status_ok; } static int reset(yajl_json_data *p, char **error_msg) { @@ -179,58 +179,58 @@ int yajl_json_init(yajl_json_data **json, char **error_msg) { static int check_utf8_byte(yajl_json_data *p, unsigned char c) { // YAJL 2.1's native check accepts overlong encodings and surrogate // codepoints; enforce Unicode scalar UTF-8 independently across chunks. - if (p->utf8_remaining) { - if (c < p->utf8_min || c > p->utf8_max) + if (p->state.utf8_remaining) { + if (c < p->state.utf8_min || c > p->state.utf8_max) return reject(p, "Invalid UTF-8 sequence"); - --p->utf8_remaining; - p->utf8_min = 0x80; - p->utf8_max = 0xbf; + --p->state.utf8_remaining; + p->state.utf8_min = 0x80; + p->state.utf8_max = 0xbf; return 1; } if (c < 0x80) return 1; - p->utf8_min = 0x80; - p->utf8_max = 0xbf; + p->state.utf8_min = 0x80; + p->state.utf8_max = 0xbf; if (c >= 0xc2 && c <= 0xdf) { - p->utf8_remaining = 1; + p->state.utf8_remaining = 1; return 1; } if (c >= 0xe0 && c <= 0xef) { - p->utf8_remaining = 2; - if (c == 0xe0) p->utf8_min = 0xa0; - if (c == 0xed) p->utf8_max = 0x9f; + p->state.utf8_remaining = 2; + if (c == 0xe0) p->state.utf8_min = 0xa0; + if (c == 0xed) p->state.utf8_max = 0x9f; return 1; } if (c >= 0xf0 && c <= 0xf4) { - p->utf8_remaining = 3; - if (c == 0xf0) p->utf8_min = 0x90; - if (c == 0xf4) p->utf8_max = 0x8f; + p->state.utf8_remaining = 3; + if (c == 0xf0) p->state.utf8_min = 0x90; + if (c == 0xf4) p->state.utf8_max = 0x8f; return 1; } return reject(p, "Invalid UTF-8 sequence"); } static void start_unicode_escape(yajl_json_data *p) { - p->escape_state = 3; - p->escape_digits = 0; - p->escape_codepoint = 0; + p->state.escape_state = 3; + p->state.escape_digits = 0; + p->state.escape_codepoint = 0; } static int finish_unicode_escape(yajl_json_data *p) { - if (p->low_surrogate_required) { - if (p->escape_codepoint < 0xdc00 || p->escape_codepoint > 0xdfff) + if (p->state.low_surrogate_required) { + if (p->state.escape_codepoint < 0xdc00 || p->state.escape_codepoint > 0xdfff) return reject(p, "Invalid Unicode surrogate escape"); - p->low_surrogate_required = 0; - p->escape_state = 1; + p->state.low_surrogate_required = 0; + p->state.escape_state = 1; return 1; } - if (p->escape_codepoint >= 0xd800 && p->escape_codepoint <= 0xdbff) { - p->low_surrogate_required = 1; - p->escape_state = 4; + if (p->state.escape_codepoint >= 0xd800 && p->state.escape_codepoint <= 0xdbff) { + p->state.low_surrogate_required = 1; + p->state.escape_state = 4; return 1; } - if (p->escape_codepoint >= 0xdc00 && p->escape_codepoint <= 0xdfff) + if (p->state.escape_codepoint >= 0xdc00 && p->state.escape_codepoint <= 0xdfff) return reject(p, "Invalid Unicode surrogate escape"); - p->escape_state = 1; + p->state.escape_state = 1; return 1; } @@ -240,29 +240,29 @@ static int check_unicode_digit(yajl_json_data *p, unsigned char c) { else if (c >= 'a' && c <= 'f') digit = c - 'a' + 10; else if (c >= 'A' && c <= 'F') digit = c - 'A' + 10; else return reject(p, "Invalid Unicode escape"); - p->escape_codepoint = p->escape_codepoint * 16 + digit; - ++p->escape_digits; - if (p->escape_digits < 4) return 1; + p->state.escape_codepoint = p->state.escape_codepoint * 16 + digit; + ++p->state.escape_digits; + if (p->state.escape_digits < 4) return 1; return finish_unicode_escape(p); } static int check_escape_byte(yajl_json_data *p, unsigned char c) { - switch (p->escape_state) { + switch (p->state.escape_state) { case 1: if (c == '"') { - p->escape_state = 0; - p->token_length = 0; - } else if (c == '\\') p->escape_state = 2; + p->state.escape_state = 0; + p->state.token_length = 0; + } else if (c == '\\') p->state.escape_state = 2; break; case 2: if (c == 'u') start_unicode_escape(p); - else p->escape_state = 1; + else p->state.escape_state = 1; break; case 3: return check_unicode_digit(p, c); case 4: if (c != '\\') return reject(p, "Invalid Unicode surrogate escape"); - p->escape_state = 5; + p->state.escape_state = 5; break; case 5: if (c != 'u') return reject(p, "Invalid Unicode surrogate escape"); @@ -273,24 +273,24 @@ static int check_escape_byte(yajl_json_data *p, unsigned char c) { } static int check_token_byte(yajl_json_data *p, unsigned char c, size_t budget) { - if (p->escape_state) { - ++p->token_length; - if (p->token_length > budget) return reject(p, "JSON token allocation limit exceeded"); + if (p->state.escape_state) { + ++p->state.token_length; + if (p->state.token_length > budget) return reject(p, "JSON token allocation limit exceeded"); return check_escape_byte(p, c); } if (c == '"') { - p->number_token = 0; - p->escape_state = 1; - p->token_length = 1; + p->state.number_token = 0; + p->state.escape_state = 1; + p->state.token_length = 1; return 1; } int number_char = (c >= '0' && c <= '9') || c == '-' || c == '+' || c == '.' || c == 'e' || c == 'E'; - if (p->number_token && number_char) { - ++p->token_length; - if (p->token_length > budget) return reject(p, "JSON token allocation limit exceeded"); + if (p->state.number_token && number_char) { + ++p->state.token_length; + if (p->state.token_length > budget) return reject(p, "JSON token allocation limit exceeded"); } else { - p->number_token = (c >= '0' && c <= '9') || c == '-'; - p->token_length = p->number_token ? 1 : 0; + p->state.number_token = (c >= '0' && c <= '9') || c == '-'; + p->state.token_length = p->state.number_token ? 1 : 0; } return 1; } @@ -305,7 +305,7 @@ static int check_input(yajl_json_data *p, const char *buf, size_t length) { return 1; } static int status_result(yajl_json_data *p, const char *buf, size_t length, char **error_msg) { - if (p->callback_error) *error_msg = strdup(p->callback_error); + if (p->state.callback_error) *error_msg = strdup(p->state.callback_error); else if (p->status != yajl_status_ok) { unsigned char *message = yajl_get_error(p->handle, 0, (const unsigned char *)buf, length); *error_msg = strdup(message ? (const char *)message : "YAJL parsing failed"); @@ -317,8 +317,8 @@ int yajl_json_process_chunk(yajl_json_data *p, const char *buf, unsigned int len if (!error_msg) return -1; *error_msg = NULL; if (!p || !p->handle || (!buf && length)) return -1; - if (p->completed) { *error_msg = strdup("YAJL document is already finalized"); return -1; } - if (p->callback_error || !check_input(p, buf, length)) return status_result(p, buf, length, error_msg); + if (p->state.completed) { *error_msg = strdup("YAJL document is already finalized"); return -1; } + if (p->state.callback_error || !check_input(p, buf, length)) return status_result(p, buf, length, error_msg); p->status = yajl_parse(p->handle, (const unsigned char *)(buf ? buf : ""), length); return status_result(p, buf, length, error_msg); } @@ -329,14 +329,14 @@ int yajl_json_complete(yajl_json_data *p, char **error_msg) { if (!error_msg) return -1; *error_msg = NULL; if (!p || !p->handle) return -1; - if (p->completed) { *error_msg = strdup("YAJL document is already finalized"); return -1; } - if (p->callback_error) return status_result(p, "", 0, error_msg); - if (p->utf8_remaining) { + if (p->state.completed) { *error_msg = strdup("YAJL document is already finalized"); return -1; } + if (p->state.callback_error) return status_result(p, "", 0, error_msg); + if (p->state.utf8_remaining) { reject(p, "Truncated UTF-8 sequence"); return status_result(p, "", 0, error_msg); } p->status = yajl_complete_parse(p->handle); - p->completed = 1; + p->state.completed = 1; if (status_result(p, "", 0, error_msg) < 0) return -1; if (!p->silence && fflush(stdout) != 0) { *error_msg = strdup("Unable to write JSON argument output"); @@ -347,8 +347,8 @@ int yajl_json_complete(yajl_json_data *p, char **error_msg) { int yajl_json_cleanup(yajl_json_data *p) { if (!p) return 1; if (p->handle) yajl_free(p->handle); - for (size_t i = 0; i < p->frame_count; ++i) free(p->frames[i].key); - free(p->frames); + for (size_t i = 0; i < p->state.frame_count; ++i) free(p->state.frames[i].key); + free(p->state.frames); free(p->prefix); free(p->current_key); free(p); diff --git a/src/yajlparser.h b/src/yajlparser.h index 5cb9a75..538fa93 100644 --- a/src/yajlparser.h +++ b/src/yajlparser.h @@ -10,6 +10,26 @@ typedef struct yajl_path_frame yajl_path_frame; #include +/* Internal structural path boundaries and incremental lexical guards. Keep + * this state inline so parser creation and cleanup need no extra allocation. + */ +typedef struct yajl_parser_state { + yajl_path_frame *frames; + size_t frame_count; + size_t frame_capacity; + const char *callback_error; + unsigned int escape_state; + unsigned int escape_digits; + unsigned int escape_codepoint; + int low_surrogate_required; + size_t token_length; + int number_token; + int completed; + unsigned int utf8_remaining; + unsigned int utf8_min; + unsigned int utf8_max; +} yajl_parser_state; + struct yajl_json_data { /* yajl configuration and parser state */ yajl_handle handle; @@ -28,21 +48,7 @@ struct yajl_json_data { long int arg_num_limit; // cppcheck-suppress unusedStructMember int arg_num_limit_exceeded; // cppcheck-suppress unusedStructMember int silence; // cppcheck-suppress unusedStructMember - /* Structural path boundaries and incremental lexical guards. */ - yajl_path_frame *frames; - size_t frame_count; - size_t frame_capacity; - const char *callback_error; - unsigned int escape_state; - unsigned int escape_digits; - unsigned int escape_codepoint; - int low_surrogate_required; - size_t token_length; - int number_token; - int completed; - unsigned int utf8_remaining; - unsigned int utf8_min; - unsigned int utf8_max; + yajl_parser_state state; }; #ifdef __cplusplus diff --git a/tests/jsoncons_api_tests.cpp b/tests/jsoncons_api_tests.cpp index b53d127..005a8b5 100644 --- a/tests/jsoncons_api_tests.cpp +++ b/tests/jsoncons_api_tests.cpp @@ -96,7 +96,7 @@ void limit_setters(Checks& checks) { checks.check("set depth to two", jsoncons_set_max_depth(parser.value, 2) == 0); checks.check("set arguments to two", jsoncons_set_max_arg_num(parser.value, 2) == 0); checks.check("enable silence", jsoncons_set_silence(parser.value, 1) == 0); - const double upper = static_cast(UINT_MAX); + const auto upper = static_cast(UINT_MAX); const std::vector> invalid = { {"NaN", std::numeric_limits::quiet_NaN()}, {"positive infinity", std::numeric_limits::infinity()}, diff --git a/tests/pathhandler_tests.cpp b/tests/pathhandler_tests.cpp index c326334..a681356 100644 --- a/tests/pathhandler_tests.cpp +++ b/tests/pathhandler_tests.cpp @@ -157,7 +157,7 @@ void boundaries_and_errors(Checks& checks) { const std::string longer_child = child + "c"; too_long.setKey(longer_child.data(), longer_child.size()); checks.check("full path at byte limit rejected", too_long.name().empty() && too_long.m_error); - std::vector untouched(limit, '!'); + std::vector untouched(limit, '!'); too_long.buildArgName(untouched.data()); checks.check("errored legacy conversion writes no bytes", untouched == std::vector(limit, '!')); diff --git a/tests/test_support.h b/tests/test_support.h index 8fe6511..9080d5e 100644 --- a/tests/test_support.h +++ b/tests/test_support.h @@ -17,6 +17,9 @@ namespace jsonbench::test { +inline constexpr auto close_c_file = [](FILE* file) noexcept { std::fclose(file); }; +inline constexpr auto free_c_diagnostic = [](char* value) noexcept { std::free(value); }; + class TestError final : public std::runtime_error { public: using std::runtime_error::runtime_error; @@ -44,7 +47,12 @@ class FileDescriptor { struct DirectoryCleanup { std::string path; + explicit DirectoryCleanup(std::string value) : path(std::move(value)) {} ~DirectoryCleanup() { ::rmdir(path.c_str()); } + DirectoryCleanup(const DirectoryCleanup&) = delete; + DirectoryCleanup& operator=(const DirectoryCleanup&) = delete; + DirectoryCleanup(DirectoryCleanup&&) = delete; + DirectoryCleanup& operator=(DirectoryCleanup&&) = delete; }; class PrivateWorkspace { @@ -81,11 +89,16 @@ inline const PrivateWorkspace& workspace() { struct FileCleanup { std::string path; bool owned = false; + explicit FileCleanup(std::string value) : path(std::move(value)) {} ~FileCleanup() { if (owned) ::unlink(path.c_str()); } + FileCleanup(const FileCleanup&) = delete; + FileCleanup& operator=(const FileCleanup&) = delete; + FileCleanup(FileCleanup&&) = delete; + FileCleanup& operator=(FileCleanup&&) = delete; }; class TemporaryFile { - FileCleanup file_{workspace().path() + "/fixture-XXXXXX", false}; + FileCleanup file_{workspace().path() + "/fixture-XXXXXX"}; FileDescriptor descriptor_; public: TemporaryFile() { @@ -121,9 +134,8 @@ class TemporaryInput { }; class CapturedOutput { - using FileCloser = int (*)(FILE*); TemporaryFile temporary_; - std::unique_ptr file_{nullptr, &std::fclose}; + std::unique_ptr file_{nullptr, close_c_file}; FileDescriptor saved_; bool active_ = false; public: @@ -145,7 +157,7 @@ class CapturedOutput { } CapturedOutput(const CapturedOutput&) = delete; CapturedOutput& operator=(const CapturedOutput&) = delete; - std::string read() { + std::string read() const { if (std::fflush(stdout) != 0 || std::fseek(file_.get(), 0, SEEK_SET) != 0) throw TestError("captured output rewind failed"); std::string result; @@ -159,9 +171,9 @@ class CapturedOutput { }; class ErrorSlot { - using Diagnostic = std::unique_ptr; - Diagnostic original_{::strdup(""), &std::free}; - Diagnostic returned_{nullptr, &std::free}; + using Diagnostic = std::unique_ptr; + Diagnostic original_{::strdup(""), free_c_diagnostic}; + Diagnostic returned_{nullptr, free_c_diagnostic}; char* value_ = original_.get(); public: ErrorSlot() { diff --git a/tests/yajl_api_tests.cpp b/tests/yajl_api_tests.cpp index 718fe06..5bef024 100644 --- a/tests/yajl_api_tests.cpp +++ b/tests/yajl_api_tests.cpp @@ -109,6 +109,26 @@ void invalid_arguments(Checks& checks) { checks.success("parser usable after invalid calls", parse(parser.value, "null"), ": \n"); } +bool status_and_diagnostic_agree(int status, const ErrorSlot& error) { + return (status == 1 && !error.get()) || + (status == -1 && error.get() && *error.get()); +} + +void check_finalized_parser(Checks& checks, yajl_json_data* parser, + std::string_view label, std::string_view expected, const CapturedOutput& output) { + ErrorSlot complete_error; + checks.check(std::string(label) + ": rejects duplicate completion", + yajl_json_complete(parser, complete_error.slot()) == -1); + checks.check(std::string(label) + ": duplicate completion has fresh diagnostic", + complete_error.replaced() && complete_error.get() && *complete_error.get()); + ErrorSlot chunk_error; + checks.check(std::string(label) + ": rejects chunks after completion", + yajl_json_process_chunk(parser, " ", 1, chunk_error.slot()) == -1); + checks.check(std::string(label) + ": post-completion chunk has fresh diagnostic", + chunk_error.replaced() && chunk_error.get() && *chunk_error.get()); + checks.check(std::string(label) + ": failed repeated calls do not emit more output", output.read() == expected); +} + void split_case(Checks& checks, std::string_view name, std::string_view input, bool valid, std::string_view expected = "") { for (std::size_t split = 0; split <= input.size(); ++split) { @@ -117,7 +137,8 @@ void split_case(Checks& checks, std::string_view name, std::string_view input, parser.value->silence = valid ? 0 : 1; CapturedOutput output; bool failed = false; - for (unsigned int part = 0; part < 2 && !failed; ++part) { + for (const unsigned int part : std::array{0u, 1u}) { + if (failed) break; const std::size_t offset = part ? split : 0; const std::size_t length = part ? input.size() - split : split; ErrorSlot error; @@ -128,7 +149,7 @@ void split_case(Checks& checks, std::string_view name, std::string_view input, static_cast(length), error.slot()); checks.check(std::string(label) + ": chunk replaces old diagnostic", error.replaced()); checks.check(std::string(label) + ": chunk result and diagnostic agree", - (rc == 1 && !error.get()) || (rc == -1 && error.get() && *error.get())); + status_and_diagnostic_agree(rc, error)); failed = rc != 1; } if (!failed) { @@ -136,24 +157,12 @@ void split_case(Checks& checks, std::string_view name, std::string_view input, const int rc = yajl_json_complete(parser.value, error.slot()); checks.check(std::string(label) + ": complete replaces old diagnostic", error.replaced()); checks.check(std::string(label) + ": complete result and diagnostic agree", - (rc == 1 && !error.get()) || (rc == -1 && error.get() && *error.get())); + status_and_diagnostic_agree(rc, error)); failed = rc != 1; } checks.check(std::string(label) + ": expected acceptance", failed != valid); checks.check(std::string(label) + ": byte-exact output", output.read() == (valid ? expected : "")); - if (valid) { - ErrorSlot complete_error; - checks.check(std::string(label) + ": rejects duplicate completion", - yajl_json_complete(parser.value, complete_error.slot()) == -1); - checks.check(std::string(label) + ": duplicate completion has fresh diagnostic", - complete_error.replaced() && complete_error.get() && *complete_error.get()); - ErrorSlot chunk_error; - checks.check(std::string(label) + ": rejects chunks after completion", - yajl_json_process_chunk(parser.value, " ", 1, chunk_error.slot()) == -1); - checks.check(std::string(label) + ": post-completion chunk has fresh diagnostic", - chunk_error.replaced() && chunk_error.get() && *chunk_error.get()); - checks.check(std::string(label) + ": failed repeated calls do not emit more output", output.read() == expected); - } + if (valid) check_finalized_parser(checks, parser.value, label, expected, output); } } @@ -168,7 +177,7 @@ void chunk_boundaries(Checks& checks) { split_case(checks, "escaped quote and slash", R"({"quote":"a\"b\\c"})", true, "quote: a\"b\\c\n"); constexpr auto nul_output = "a\0b: x\0y\n"sv; split_case(checks, "decoded NUL", R"({"a\u0000b":"x\u0000y"})", true, - std::string(nul_output)); + nul_output); split_case(checks, "root and nested arrays", R"([[null,true],{"k":false},2])", true, "array.array.array: \narray.array.array: true\narray.array.k: false\narray.array: 2\n"); split_case(checks, "dotted parents and siblings", R"({"a.b":[{"c.d":1},2],"after":3})", true, @@ -344,7 +353,7 @@ void length_boundaries(Checks& checks) { constexpr auto expected = "a\0b.c\0d: v\0x\na\0b.after: false\nlast: \n"sv; checks.success("NUL parents restore with explicit lengths", parse(parser.value, R"({"a\u0000b":{"c\u0000d":"v\u0000x","after":false},"last":null})"), - std::string(expected)); + expected); checks.success("dotted keys are not structural pop delimiters", parse(parser.value, R"({"a.b":{"c.d":1},"after":2})"), "a.b.c.d: 1\nafter: 2\n"); } From bdeb8e462f41238de4ac109a9d7e555540e53594 Mon Sep 17 00:00:00 2001 From: jens Steinbacher Date: Tue, 13 Oct 2026 17:41:27 +0000 Subject: [PATCH 09/18] Clarify streaming captures and typed scratch storage --- src/nlparser.cpp | 6 +++--- src/rjparser.cpp | 19 ++++++++++--------- tests/sax_adapter_api_tests.cpp | 3 ++- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/nlparser.cpp b/src/nlparser.cpp index 7925a3f..8ec945e 100644 --- a/src/nlparser.cpp +++ b/src/nlparser.cpp @@ -245,7 +245,7 @@ extern "C" int nl_json_init(nl_parser** parser, char** error_msg) { if (error_msg) *error_msg = nullptr; if (parser) *parser = nullptr; if (!parser || !error_msg) return -1; - return at_c_boundary([&] { + return at_c_boundary([parser] { *parser = std::make_unique().release(); return 0; }, error_msg, -1); @@ -261,7 +261,7 @@ extern "C" int nl_parse_buffer(nl_parser* parser, const char* buffer, if (error_msg) *error_msg = nullptr; if (!parser || !buffer || !error_msg) return fail(error_msg, "Invalid NlohmannJSON parser arguments", -1); - return at_c_boundary([&] { + return at_c_boundary([parser, buffer, length, error_msg] { CheckedInputBuffer source(buffer, length); return parse(parser, source, error_msg); }, error_msg); @@ -271,7 +271,7 @@ extern "C" int nl_parse_file(nl_parser* parser, const char* filename, char** err if (error_msg) *error_msg = nullptr; if (!parser || !filename || !error_msg) return fail(error_msg, "Invalid NlohmannJSON parser arguments", -1); - return at_c_boundary([&] { + return at_c_boundary([parser, filename, error_msg] { const std::unique_ptr file(std::fopen(filename, "rb")); if (!file) return fail(error_msg, "Unable to open JSON file", -1); CheckedInputBuffer source(file.get()); diff --git a/src/rjparser.cpp b/src/rjparser.cpp index 8f07da0..34008c4 100644 --- a/src/rjparser.cpp +++ b/src/rjparser.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -46,16 +47,16 @@ class TemporaryAllocator { if (size > budget) throw RapidJsonInputError("RapidJSON token/state allocation limit exceeded"); // RapidJSON 1.1's GenericReader stack passes char* storage to Realloc - // and Free. A char array supplies alignment for its scalar state values. + // and Free. Byte-array allocation supplies alignment for its scalar states. // Allocate first: an exception leaves the original stack allocation live. - auto replacement = size == 0 ? nullptr : std::make_unique(size); + auto replacement = size == 0 ? nullptr : std::make_unique(size); if (data != nullptr && size != 0) - std::copy_n(data, std::min(old_size, size), replacement.get()); - const std::unique_ptr original(data); - return replacement.release(); + std::memcpy(replacement.get(), data, std::min(old_size, size)); + const std::unique_ptr original(reinterpret_cast(data)); + return reinterpret_cast(replacement.release()); } static void Free(char* data) noexcept { - const std::unique_ptr allocation(data); + const std::unique_ptr allocation(reinterpret_cast(data)); } }; @@ -254,7 +255,7 @@ extern "C" int rj_json_init(rj_parser** parser, char** error_msg) { *error_msg = nullptr; if (parser == nullptr) return -1; *parser = nullptr; - return at_c_boundary([&] { + return at_c_boundary([parser] { *parser = std::make_unique().release(); return 0; }, error_msg, 1); @@ -265,7 +266,7 @@ extern "C" int rj_parse_buffer(rj_parser* parser, const char* buf, if (error_msg == nullptr) return -1; *error_msg = nullptr; if (parser == nullptr || buf == nullptr) return -1; - return at_c_boundary([&] { + return at_c_boundary([parser, buf, len, error_msg] { rapidjson::MemoryStream input(buf, len); return parse(parser, input, error_msg); }, error_msg); @@ -276,7 +277,7 @@ extern "C" int rj_parse_file(rj_parser* parser, const char* filename, if (error_msg == nullptr) return -1; *error_msg = nullptr; if (parser == nullptr || filename == nullptr) return -1; - return at_c_boundary([&] { + return at_c_boundary([parser, filename, error_msg] { const std::unique_ptr file(std::fopen(filename, "rb")); if (!file) { *error_msg = strdup("Unable to open JSON file"); diff --git a/tests/sax_adapter_api_tests.cpp b/tests/sax_adapter_api_tests.cpp index 94d16b2..43e981b 100644 --- a/tests/sax_adapter_api_tests.cpp +++ b/tests/sax_adapter_api_tests.cpp @@ -97,8 +97,9 @@ void invalid_arguments(Checks& checks) { TemporaryInput input("null"); for (unsigned int kind = 0; kind < 3; ++kind) { ErrorSlot error; + const unsigned int length = kind == 1 ? 0 : 1; const int rc = kind == 0 ? Api::buffer(nullptr, "{}", 2, error.slot()) : - Api::buffer(parser.value, nullptr, kind == 1 ? 0 : 1, error.slot()); + Api::buffer(parser.value, nullptr, length, error.slot()); checks.check("invalid buffer arguments " + std::to_string(kind), rc != 0); checks.check("invalid buffer replaces diagnostic " + std::to_string(kind), error.replaced()); checks.check("invalid buffer diagnostic is null or nonempty " + std::to_string(kind), From 6917ba4c44931cb10e8aa22feb444423d86e7c9f Mon Sep 17 00:00:00 2001 From: jens Steinbacher Date: Tue, 13 Oct 2026 17:41:27 +0000 Subject: [PATCH 10/18] Simplify native input policy and validation helpers --- src/cjsonparser.c | 5 ++-- src/domarguments.h | 46 +++++++++++++++++----------------- src/janssonparser.c | 5 ++-- src/jsoncparser.c | 57 ++++++++++++++++++++++++------------------- src/jsoncppparser.cpp | 27 ++++++++++++++------ src/yyjsonparser.c | 4 +-- 6 files changed, 83 insertions(+), 61 deletions(-) diff --git a/src/cjsonparser.c b/src/cjsonparser.c index 304d3f1..d1b401e 100644 --- a/src/cjsonparser.c +++ b/src/cjsonparser.c @@ -42,6 +42,7 @@ static int cjson_scalar(void *node, dom_value *value, char **error) { } static const dom_adapter cjson_adapter = {cjson_kind, cjson_child, cjson_scalar}; +static const dom_input_policy cjson_policy = {DOM_NUL_FORBIDDEN, DOM_NUM_DOUBLE}; int cjson_parse_file(cjson_parser *parser, const char *filename, char **error_msg) { char *content = NULL; @@ -61,8 +62,8 @@ int cjson_parse_file(cjson_parser *parser, const char *filename, char **error_ms if (rc != 0) goto done; /* cJSON has no decoded string lengths and stores all numbers as doubles. * Reject NUL and unsafe integers instead of silently truncating/rounding. */ - rc = dom_preflight_numbers(content, length, &parser->settings, DOM_NUL_FORBIDDEN, - DOM_NUM_DOUBLE, &keys, &numbers, error_msg); + rc = dom_preflight_numbers(content, length, &parser->settings, &cjson_policy, + &keys, &numbers, error_msg); if (rc != 0) goto done; root = cJSON_ParseWithLengthOpts(content, length + 1, &end, 1); if (root == NULL) { diff --git a/src/domarguments.h b/src/domarguments.h index a232569..5f484ce 100644 --- a/src/domarguments.h +++ b/src/domarguments.h @@ -153,9 +153,8 @@ static int dom_hex(unsigned char c) { static int dom_code_unit(const char *data, size_t length, size_t *position, unsigned int *code) { unsigned int result = 0; - size_t i; if (length - *position < 4) return 0; - for (i = 0; i < 4; ++i) { + for (size_t i = 0; i < 4; ++i) { int digit = dom_hex((unsigned char)data[(*position)++]); if (digit < 0) return 0; result = result * 16 + (unsigned int)digit; @@ -168,7 +167,6 @@ static int dom_code_unit(const char *data, size_t length, size_t *position, static int dom_utf8(const char *data, size_t length, size_t *position) { unsigned char first = (unsigned char)data[*position]; size_t count; - size_t i; unsigned int code; unsigned int minimum; if (first < 0x80) { ++*position; return 1; } @@ -177,7 +175,7 @@ static int dom_utf8(const char *data, size_t length, size_t *position) { else if (first >= 0xf0 && first <= 0xf4) { count = 4; code = first & 7; minimum = 0x10000; } else return 0; if (length - *position < count) return 0; - for (i = 1; i < count; ++i) { + for (size_t i = 1; i < count; ++i) { unsigned char byte = (unsigned char)data[*position + i]; if ((byte & 0xc0) != 0x80) return 0; code = code * 64 + (byte & 63); @@ -191,6 +189,12 @@ enum { DOM_NUL_ALLOWED = 0, DOM_NUL_KEY_FORBIDDEN = 1, DOM_NUL_FORBIDDEN = 2 }; enum { DOM_NUM_RAW = 0, DOM_NUM_INT64_UINT64 = 1, DOM_NUM_DOUBLE = 2, DOM_NUM_SIGNED = 3 }; enum { DOM_NOT_NUMBER = 0, DOM_INTEGER = 1, DOM_REAL = 2, DOM_ANY_NUMBER = 3 }; +/* Native representation limits belong together; settings hold user limits. */ +typedef struct { + int nul_policy; + int number_policy; +} dom_input_policy; + typedef struct { const char *data; size_t length; @@ -503,15 +507,15 @@ static int dom_scan_token(dom_scan *scan) { } static int dom_preflight_impl(const char *data, size_t length, const dom_settings *settings, - int nul_policy, int number_policy, size_t *keys, + const dom_input_policy *policy, size_t *keys, dom_numbers *numbers, char **error) { dom_scan scan; memset(&scan, 0, sizeof(scan)); scan.data = data; scan.length = length; scan.settings = settings; - scan.nul_policy = nul_policy; - scan.number_policy = number_policy; + scan.nul_policy = policy->nul_policy; + scan.number_policy = policy->number_policy; scan.keys = keys; scan.numbers = numbers; scan.error = error; @@ -524,14 +528,14 @@ static int dom_preflight_impl(const char *data, size_t length, const dom_setting } static inline int dom_preflight(const char *data, size_t length, const dom_settings *settings, - int nul_policy, int number_policy, size_t *keys, char **error) { + const dom_input_policy *policy, size_t *keys, char **error) { dom_locale_guard locale = {0}; int result; - if (number_policy == DOM_NUM_RAW) - return dom_preflight_impl(data, length, settings, nul_policy, number_policy, keys, NULL, error); + if (policy->number_policy == DOM_NUM_RAW) + return dom_preflight_impl(data, length, settings, policy, keys, NULL, error); result = dom_locale_begin(&locale, error); if (result != 0) return result; - result = dom_preflight_impl(data, length, settings, nul_policy, number_policy, keys, NULL, error); + result = dom_preflight_impl(data, length, settings, policy, keys, NULL, error); return dom_locale_finish(&locale, result, error); } @@ -540,16 +544,15 @@ static inline int dom_preflight(const char *data, size_t length, const dom_setti * number kind and total count, including during silent validation. Native * range checks remain in effect, even though output avoids binary reformatting. */ static inline int dom_preflight_numbers(const char *data, size_t length, - const dom_settings *settings, int nul_policy, - int number_policy, size_t *keys, - dom_numbers *numbers, char **error) { + const dom_settings *settings, const dom_input_policy *policy, + size_t *keys, dom_numbers *numbers, char **error) { dom_locale_guard locale = {0}; int result; - if (number_policy == DOM_NUM_RAW) - return dom_preflight_impl(data, length, settings, nul_policy, number_policy, keys, numbers, error); + if (policy->number_policy == DOM_NUM_RAW) + return dom_preflight_impl(data, length, settings, policy, keys, numbers, error); result = dom_locale_begin(&locale, error); if (result != 0) return result; - result = dom_preflight_impl(data, length, settings, nul_policy, number_policy, keys, numbers, error); + result = dom_preflight_impl(data, length, settings, policy, keys, numbers, error); return dom_locale_finish(&locale, result, error); } @@ -696,8 +699,7 @@ static int dom_walk_write(const dom_walk_state *state, const dom_value *value) { {value->data, value->length}, {"\n", 1} }; - size_t i; - for (i = 0; i < sizeof(parts) / sizeof(parts[0]); ++i) { + for (size_t i = 0; i < sizeof(parts) / sizeof(parts[0]); ++i) { if (parts[i].length == 0) continue; if (fwrite(parts[i].data, 1, parts[i].length, stdout) != parts[i].length) return dom_error(state->error, "Unable to write JSON argument output"); @@ -783,11 +785,9 @@ static int dom_walk_impl(void *root, const dom_adapter *adapter, state.object_keys = object_keys; state.error = error; *object_keys = 0; - while (entering) { + while (entering && result == 0) { result = dom_walk_enter(&state); - if (result != 0) break; - result = dom_walk_next(&state, &entering); - if (result != 0) break; + if (result == 0) result = dom_walk_next(&state, &entering); } return dom_walk_finish(&state, result); } diff --git a/src/janssonparser.c b/src/janssonparser.c index 9bccc79..b89d445 100644 --- a/src/janssonparser.c +++ b/src/janssonparser.c @@ -53,6 +53,7 @@ static int jansson_scalar(void *node, dom_value *value, char **error) { } static const dom_adapter jansson_adapter = {jansson_kind, jansson_child, jansson_scalar}; +static const dom_input_policy jansson_policy = {DOM_NUL_ALLOWED, DOM_NUM_SIGNED}; int jansson_parse_file(jansson_parser *parser, const char *filename, char **error_msg) { char *content = NULL; @@ -70,8 +71,8 @@ int jansson_parse_file(jansson_parser *parser, const char *filename, char **erro if (rc != 0) return rc; rc = dom_read_file(filename, &content, &length, error_msg); if (rc != 0) goto done; - rc = dom_preflight_numbers(content, length, &parser->settings, DOM_NUL_ALLOWED, - DOM_NUM_SIGNED, &keys, &numbers, error_msg); + rc = dom_preflight_numbers(content, length, &parser->settings, &jansson_policy, + &keys, &numbers, error_msg); if (rc != 0) goto done; /* Native objects coalesce keys; fail explicitly instead of silently losing * occurrences. JSON_ALLOW_NUL preserves length-aware decoded strings. */ diff --git a/src/jsoncparser.c b/src/jsoncparser.c index 0c75eee..7c98316 100644 --- a/src/jsoncparser.c +++ b/src/jsoncparser.c @@ -56,6 +56,7 @@ static int jsonc_scalar(void *node, dom_value *value, char **error) { } static const dom_adapter jsonc_adapter = {jsonc_kind, jsonc_child, jsonc_scalar}; +static const dom_input_policy jsonc_policy = {DOM_NUL_KEY_FORBIDDEN, DOM_NUM_INT64_UINT64}; static void jsonc_unicode_escape(char *output, size_t *used, unsigned int code) { static const char hex[] = "0123456789abcdef"; @@ -95,7 +96,6 @@ static int jsonc_ascii_size(const char *data, size_t length, size_t *ascii_lengt static unsigned int jsonc_next_codepoint(const char *data, size_t *position) { unsigned int code = (unsigned char)data[*position]; size_t count; - size_t i; ++*position; if (code < 0xe0) { code &= 31; @@ -107,7 +107,7 @@ static unsigned int jsonc_next_codepoint(const char *data, size_t *position) { code &= 7; count = 3; } - for (i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { code = code * 64 + ((unsigned char)data[*position] & 63); ++*position; } @@ -141,7 +141,7 @@ static size_t jsonc_ascii_write(const char *data, size_t length, char *output) { static int jsonc_ascii_input(const char *data, size_t length, char **ascii, size_t *ascii_length, char **error) { - size_t output_length; + size_t output_length = length; size_t used; char *output; int result = jsonc_ascii_size(data, length, &output_length, error); @@ -160,6 +160,31 @@ static int jsonc_ascii_input(const char *data, size_t length, char **ascii, return 0; } +static int jsonc_native_parse(const char *input, size_t length, unsigned int max_depth, + struct json_tokener **tokener, struct json_object **root, + char **error_msg) { + size_t end; + enum json_tokener_error native_error; + if (length >= INT_MAX) return dom_error(error_msg, "JSON-C input exceeds native API range"); + /* The native tokener counts scalar slots as well as container frames. + * Bound native state allocation even when a caller sets UINT_MAX. Limits + * above 8192 do not remove this adapter's independent native stack cap. */ + if (max_depth > JSON_STRING_SIZE) max_depth = JSON_STRING_SIZE; + *tokener = json_tokener_new_ex((int)max_depth + 1); + if (*tokener == NULL) return dom_error(error_msg, "Unable to allocate JSON-C tokener"); + json_tokener_set_flags(*tokener, JSON_TOKENER_STRICT | JSON_TOKENER_VALIDATE_UTF8); + /* Include our sentinel so a root number at physical EOF is finalized. Raw + * NUL inside the actual input was rejected by the lexical boundary guard. */ + *root = json_tokener_parse_ex(*tokener, input, (int)length + 1); + native_error = json_tokener_get_error(*tokener); + if (native_error != json_tokener_success) + return dom_error(error_msg, json_tokener_error_desc(native_error)); + end = json_tokener_get_parse_end(*tokener); + while (end < length && dom_space((unsigned char)input[end])) ++end; + if (end < length) return dom_error(error_msg, "Unexpected content after JSON document"); + return 0; +} + int jsonc_parse_file(jsonc_parser *parser, const char *filename, char **error_msg) { char *content = NULL; char *ascii = NULL; @@ -167,7 +192,6 @@ int jsonc_parse_file(jsonc_parser *parser, const char *filename, char **error_ms size_t length = 0; size_t keys = 0; size_t native_keys = 0; - size_t end; dom_numbers numbers = {0}; dom_locale_guard locale = {0}; struct json_tokener *tokener = NULL; @@ -181,31 +205,14 @@ int jsonc_parse_file(jsonc_parser *parser, const char *filename, char **error_ms if (rc != 0) return rc; rc = dom_read_file(filename, &content, &length, error_msg); if (rc != 0) goto done; - rc = dom_preflight_numbers(content, length, &parser->settings, DOM_NUL_KEY_FORBIDDEN, - DOM_NUM_INT64_UINT64, &keys, &numbers, error_msg); + rc = dom_preflight_numbers(content, length, &parser->settings, &jsonc_policy, + &keys, &numbers, error_msg); if (rc != 0) goto done; rc = jsonc_ascii_input(content, length, &ascii, &length, error_msg); if (rc != 0) goto done; input = ascii != NULL ? ascii : content; - if (length >= INT_MAX) { - rc = dom_error(error_msg, "JSON-C input exceeds native API range"); goto done; - } - /* The native tokener counts scalar slots as well as container frames. - * Bound native state allocation even when a caller sets UINT_MAX. Limits - * above 8192 do not remove this adapter's independent native stack cap. */ - tokener = json_tokener_new_ex((int)(parser->settings.max_depth < JSON_STRING_SIZE ? - parser->settings.max_depth : JSON_STRING_SIZE) + 1); - if (tokener == NULL) { rc = dom_error(error_msg, "Unable to allocate JSON-C tokener"); goto done; } - json_tokener_set_flags(tokener, JSON_TOKENER_STRICT | JSON_TOKENER_VALIDATE_UTF8); - /* Include our sentinel so a root number at physical EOF is finalized. Raw - * NUL inside the actual input was rejected by the lexical boundary guard. */ - root = json_tokener_parse_ex(tokener, input, (int)length + 1); - if (json_tokener_get_error(tokener) != json_tokener_success) { - rc = dom_error(error_msg, json_tokener_error_desc(json_tokener_get_error(tokener))); goto done; - } - end = json_tokener_get_parse_end(tokener); - while (end < length && dom_space((unsigned char)input[end])) ++end; - if (end < length) { rc = dom_error(error_msg, "Unexpected content after JSON document"); goto done; } + rc = jsonc_native_parse(input, length, parser->settings.max_depth, &tokener, &root, error_msg); + if (rc != 0) goto done; /* JSON-C coalesces repeated keys. Check retained vs. input key counts before * emitting output; reject rather than silently dropping security arguments. */ check = parser->settings; check.silence = 1; diff --git a/src/jsoncppparser.cpp b/src/jsoncppparser.cpp index 8431d87..6c2623d 100644 --- a/src/jsoncppparser.cpp +++ b/src/jsoncppparser.cpp @@ -126,6 +126,22 @@ void validate_encoding(const std::string& text) { } } +bool decimal_digit(char c) { + return c >= '0' && c <= '9'; +} + +std::size_t digits_end(const std::string& token, std::size_t position) { + while (position < token.size() && decimal_digit(token[position])) ++position; + return position; +} + +bool required_digits(const std::string& token, std::size_t& position) { + const std::size_t end = digits_end(token, position); + const bool present = end != position; + position = end; + return present; +} + bool valid_number(const std::string& token) { std::size_t i = 0; if (i < token.size() && token[i] == '-') ++i; @@ -133,19 +149,16 @@ bool valid_number(const std::string& token) { if (token[i] == '0') ++i; else { if (token[i] < '1' || token[i] > '9') return false; - while (i < token.size() && token[i] >= '0' && token[i] <= '9') ++i; + i = digits_end(token, i); } if (i < token.size() && token[i] == '.') { - const std::size_t start = ++i; - while (i < token.size() && token[i] >= '0' && token[i] <= '9') ++i; - if (i == start) return false; + ++i; + if (!required_digits(token, i)) return false; } if (i < token.size() && (token[i] == 'e' || token[i] == 'E')) { ++i; if (i < token.size() && (token[i] == '+' || token[i] == '-')) ++i; - const std::size_t start = i; - while (i < token.size() && token[i] >= '0' && token[i] <= '9') ++i; - if (i == start) return false; + if (!required_digits(token, i)) return false; } return i == token.size(); } diff --git a/src/yyjsonparser.c b/src/yyjsonparser.c index 9240c17..95a5285 100644 --- a/src/yyjsonparser.c +++ b/src/yyjsonparser.c @@ -49,6 +49,7 @@ static int yyjson_scalar(void *node, dom_value *value, char **error) { } static const dom_adapter yyjson_adapter = {yyjson_kind, yyjson_child, yyjson_scalar}; +static const dom_input_policy yyjson_policy = {DOM_NUL_ALLOWED, DOM_NUM_RAW}; int yyjson_parse_file(yyjson_parser *parser, const char *filename, char **error_msg) { char *content = NULL; @@ -62,8 +63,7 @@ int yyjson_parse_file(yyjson_parser *parser, const char *filename, char **error_ if (parser == NULL || filename == NULL) return dom_error(error_msg, "Invalid yyjson parser arguments"); rc = dom_read_file(filename, &content, &length, error_msg); if (rc != 0) return rc; - rc = dom_preflight(content, length, &parser->settings, DOM_NUL_ALLOWED, - DOM_NUM_RAW, &keys, error_msg); + rc = dom_preflight(content, length, &parser->settings, &yyjson_policy, &keys, error_msg); if (rc != 0) goto done; /* Public native option preserves arbitrary JSON number lexemes and the * native object iterator retains repeated keys in original order. */ From 30281713ef4098c51f693bfc65a2eac6bb622dba Mon Sep 17 00:00:00 2001 From: jens Steinbacher Date: Tue, 13 Oct 2026 18:02:08 +0000 Subject: [PATCH 11/18] Release test diagnostics through their C allocator boundary --- tests/test_c_memory.c | 7 +++++++ tests/test_c_memory.h | 16 ++++++++++++++++ tests/test_support.h | 6 +++++- 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 tests/test_c_memory.c create mode 100644 tests/test_c_memory.h diff --git a/tests/test_c_memory.c b/tests/test_c_memory.c new file mode 100644 index 0000000..fb63e75 --- /dev/null +++ b/tests/test_c_memory.c @@ -0,0 +1,7 @@ +#include "test_c_memory.h" + +#include + +void jsonbench_test_release_diagnostic(char* memory) { + free(memory); +} diff --git a/tests/test_c_memory.h b/tests/test_c_memory.h new file mode 100644 index 0000000..7645b4a --- /dev/null +++ b/tests/test_c_memory.h @@ -0,0 +1,16 @@ +#ifndef JSONBENCH_TEST_C_MEMORY_H +#define JSONBENCH_TEST_C_MEMORY_H + +#ifdef __cplusplus +extern "C" { +#endif + +// Parser diagnostics and strdup seeds use C/POSIX allocation. Their C++ RAII +// owner delegates release to the corresponding C allocator through this bridge. +void jsonbench_test_release_diagnostic(char* memory); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/tests/test_support.h b/tests/test_support.h index 9080d5e..ab63420 100644 --- a/tests/test_support.h +++ b/tests/test_support.h @@ -1,6 +1,8 @@ #ifndef JSONBENCH_TEST_SUPPORT_H #define JSONBENCH_TEST_SUPPORT_H +#include "test_c_memory.h" + #include #include #include @@ -18,7 +20,9 @@ namespace jsonbench::test { inline constexpr auto close_c_file = [](FILE* file) noexcept { std::fclose(file); }; -inline constexpr auto free_c_diagnostic = [](char* value) noexcept { std::free(value); }; +inline constexpr auto free_c_diagnostic = [](char* value) noexcept { + jsonbench_test_release_diagnostic(value); +}; class TestError final : public std::runtime_error { public: From 8b003c9ead7b8f2c7d8d52785ec255450901028c Mon Sep 17 00:00:00 2001 From: jens Steinbacher Date: Tue, 13 Oct 2026 18:02:27 +0000 Subject: [PATCH 12/18] Document iteration validation and standalone C test helper linking --- tests/README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/README.md b/tests/README.md index 877cbea..af59f37 100644 --- a/tests/README.md +++ b/tests/README.md @@ -20,7 +20,9 @@ containers, booleans/null, Unicode, embedded NUL, escaped characters, numeric boundaries and precision, malformed JSON/UTF-8, depth and argument boundaries, long paths/keys/values, huge tokens, a document larger than the legacy file buffer, silent mode, repeated benchmark iterations, CLI limit validation, file -errors and output errors. Existing fixtures and cross-engine comparisons remain +errors and output errors. Iteration counts reject negative, zero, nonnumeric, +fractional, trailing-junk and overflowing inputs before allocation. Existing +fixtures and cross-engine comparisons remain included. New expected outputs are explicit data in the helper, not generated by the parser under test. Argument counts are also tested by running exactly at the leaf count and one below it. Boolean and null leaves count as arguments; @@ -66,13 +68,16 @@ After configuring with JSONCONS enabled, compile and run it with the same header paths used for the main build: ```sh +gcc -std=gnu11 -Wall -Wextra -Werror -c tests/test_c_memory.c \ + -o /tmp/jsonbench-test-c-memory.o g++ -std=c++17 -Wall -Wextra -Werror $CPPFLAGS \ - tests/jsoncons_api_tests.cpp src/jsonconsparser.cpp -o /tmp/jsoncons-api-tests + tests/jsoncons_api_tests.cpp src/jsonconsparser.cpp \ + /tmp/jsonbench-test-c-memory.o -o /tmp/jsoncons-api-tests /tmp/jsoncons-api-tests ``` `CPPFLAGS` must include the installed jsoncons include path if it is outside the compiler's standard paths. Add `-fsanitize=address,undefined --fno-omit-frame-pointer` to this command to run the harness with sanitizers. +-fno-omit-frame-pointer` to both compilation commands to run the harness with sanitizers. The API harness returns nonzero on failure and returns 77 if configured without JSONCONS; that is unavailable coverage, not a passing test. From 1a236bb52d3f1b76904e0be59222e8f2fc54b9be Mon Sep 17 00:00:00 2001 From: jens Steinbacher Date: Tue, 13 Oct 2026 18:04:46 +0000 Subject: [PATCH 13/18] Link C diagnostic ownership helper in documented API check --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 87fde65..34e60a4 100644 --- a/README.md +++ b/README.md @@ -194,12 +194,14 @@ To exercise the jsoncons C interface separately after configuring the project (use the same include flags as your normal build): ```bash -g++ -std=c++17 ${CPPFLAGS:-} tests/jsoncons_api_tests.cpp src/jsonconsparser.cpp -o /tmp/jsoncons_api_tests +gcc -std=gnu11 -c tests/test_c_memory.c -o /tmp/jsonbench-test-c-memory.o +g++ -std=c++17 ${CPPFLAGS:-} tests/jsoncons_api_tests.cpp src/jsonconsparser.cpp \ + /tmp/jsonbench-test-c-memory.o -o /tmp/jsoncons_api_tests /tmp/jsoncons_api_tests ``` For memory-safety validation, add `-fsanitize=address,undefined --fno-omit-frame-pointer` to that compile command. For repeatable DOM/SAX/YAJL +-fno-omit-frame-pointer` to both compilation commands. For repeatable DOM/SAX/YAJL measurements, preserve a pre-change executable and run: ```bash From f8629f106a4b64e758344c04819de9294ec884d6 Mon Sep 17 00:00:00 2001 From: jens Steinbacher Date: Tue, 13 Oct 2026 18:05:26 +0000 Subject: [PATCH 14/18] Bound RapidJSON reader scratch memory with an owned vector arena --- src/rjparser.cpp | 81 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 20 deletions(-) diff --git a/src/rjparser.cpp b/src/rjparser.cpp index 34008c4..ebb1b5e 100644 --- a/src/rjparser.cpp +++ b/src/rjparser.cpp @@ -4,10 +4,10 @@ #include "pathhandler.h" -#include #include #include #include +#include #include #include #include @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -34,29 +35,69 @@ class RapidJsonInputError final : public std::runtime_error { using std::runtime_error::runtime_error; }; -// RapidJSON keeps one decoded token and its iterative structural stack here. -// A checked allocator stops hostile unterminated strings/numbers from growing -// document-sized storage before a SAX callback can enforce the token limit. +static_assert(alignof(rapidjson::SizeType) <= alignof(std::max_align_t), + "RapidJSON state must fit fundamental allocation alignment"); + +constexpr std::size_t scratch_budget = + static_cast(JSON_STRING_SIZE) * 4 + 16384; + +// Bound the vector's actual allocation, including any implementation-specific +// reserve rounding. GenericReader also stores SizeType scalars in this storage. +template +class ScratchAllocator { + public: + using value_type = T; + ScratchAllocator() = default; + template explicit ScratchAllocator(const ScratchAllocator&) {} + T* allocate(std::size_t count) const { + if (count > scratch_budget / sizeof(T)) + throw RapidJsonInputError("RapidJSON token/state allocation limit exceeded"); + std::allocator allocator; + auto* memory = allocator.allocate(count); + // The installed standard allocator uses global operator new, which + // meets this alignment. Check it explicitly for other implementations. + if (reinterpret_cast(memory) % alignof(rapidjson::SizeType) != 0) { + allocator.deallocate(memory, count); + throw RapidJsonInputError("RapidJSON scratch storage is not aligned"); + } + return memory; + } + void deallocate(T* data, std::size_t count) const noexcept { + std::allocator{}.deallocate(data, count); + } + template bool operator==(const ScratchAllocator&) const noexcept { return true; } + template bool operator!=(const ScratchAllocator&) const noexcept { return false; } +}; + +// Each parse here creates a fresh GenericReader with its own stack allocator. +// Keep its UTF-8 token and scalar state storage in one bounded vector. Like +// RapidJSON's public MemoryPoolAllocator, Free is a no-op: ownership remains +// with the allocator, whose vector releases all storage at destruction. class TemporaryAllocator { + std::vector> storage_; public: - static constexpr bool kNeedFree = true; - char* Malloc(std::size_t size) const { return Realloc(nullptr, 0, size); } - char* Realloc(char* data, std::size_t old_size, std::size_t size) const { - constexpr std::size_t budget = - static_cast(JSON_STRING_SIZE) * 4 + 16384; - if (size > budget) + static constexpr bool kNeedFree = false; + char* Malloc(std::size_t size) { return Realloc(nullptr, 0, size); } + char* Realloc(char* data, std::size_t old_size, std::size_t size) { + if (size > scratch_budget) throw RapidJsonInputError("RapidJSON token/state allocation limit exceeded"); - // RapidJSON 1.1's GenericReader stack passes char* storage to Realloc - // and Free. Byte-array allocation supplies alignment for its scalar states. - // Allocate first: an exception leaves the original stack allocation live. - auto replacement = size == 0 ? nullptr : std::make_unique(size); - if (data != nullptr && size != 0) - std::memcpy(replacement.get(), data, std::min(old_size, size)); - const std::unique_ptr original(reinterpret_cast(data)); - return reinterpret_cast(replacement.release()); + if ((data == nullptr && old_size != 0) || + (data != nullptr && (data != storage_.data() || old_size != storage_.size())) || + (data == nullptr && !storage_.empty())) + throw RapidJsonInputError("Invalid RapidJSON scratch allocation"); + if (size == 0) { + storage_.clear(); + return nullptr; + } + // reserve has the strong exception guarantee for char; a failed growth + // leaves the reader's old pointer and bytes valid. Reserve first so + // resize cannot request its usual geometric growth beyond the budget. + storage_.reserve(size); + storage_.resize(size); + return storage_.data(); } - static void Free(char* data) noexcept { - const std::unique_ptr allocation(reinterpret_cast(data)); + static void Free(char*) noexcept { + // The owning vector is released when GenericReader deletes its allocator. } }; From 4ed912228c0d881070d2ea9bf040270f481849f9 Mon Sep 17 00:00:00 2001 From: jens Steinbacher Date: Tue, 13 Oct 2026 18:06:10 +0000 Subject: [PATCH 15/18] Share numeric DOM initialization and integrate C diagnostic test ownership --- docs/adapter-stack.md | 73 ++++++++++++++++++++++++++++++++----------- src/cjsonparser.c | 8 ++--- src/domarguments.h | 17 ++++++++++ src/janssonparser.c | 8 ++--- src/jsoncparser.c | 8 ++--- tests/Makefile.am | 12 +++---- tests/README.md | 6 ++++ 7 files changed, 89 insertions(+), 43 deletions(-) diff --git a/docs/adapter-stack.md b/docs/adapter-stack.md index bbb7443..a0e8b1d 100644 --- a/docs/adapter-stack.md +++ b/docs/adapter-stack.md @@ -43,12 +43,15 @@ source /workspace/.jsonbench-tools/activate.sh ./autogen.sh && ./configure && make -j2 tests/run_tests.sh YAJL # Standalone public API checks (use the configured include/link flags): +gcc -std=gnu11 -Wall -Wextra -Werror -c tests/test_c_memory.c \ + -o /tmp/jsonbench-test-c-memory.o gcc -std=gnu11 -Wall -Wextra -Werror $CPPFLAGS -c src/yajlparser.c -o /tmp/yajl.o g++ -std=c++17 -Wall -Wextra -Werror $CPPFLAGS \ - tests/yajl_api_tests.cpp /tmp/yajl.o $LDFLAGS -lyajl -o /tmp/yajl-api-tests + tests/yajl_api_tests.cpp /tmp/yajl.o /tmp/jsonbench-test-c-memory.o \ + $LDFLAGS -lyajl -o /tmp/yajl-api-tests /tmp/yajl-api-tests g++ -std=c++17 -Wall -Wextra -Werror $CPPFLAGS \ - tests/pathhandler_tests.cpp -o /tmp/pathhandler-tests + tests/pathhandler_tests.cpp /tmp/jsonbench-test-c-memory.o -o /tmp/pathhandler-tests /tmp/pathhandler-tests ``` @@ -90,7 +93,7 @@ detection. Either engine can be compiled alone; both disabled return skip code ```sh g++ -std=c++17 -Wall -Wextra -Werror $CPPFLAGS -I. \ tests/sax_adapter_api_tests.cpp src/rjparser.cpp src/nlparser.cpp \ - $LDFLAGS -o /tmp/sax-api-tests + /tmp/jsonbench-test-c-memory.o $LDFLAGS -o /tmp/sax-api-tests /tmp/sax-api-tests ``` @@ -150,23 +153,23 @@ all native representation limitations visible: | Engine | Passed | Failed | Remaining required failures | | --- | ---: | ---: | --- | -| YAJL | 235 | 0 | None | -| RapidJSON | 231 | 4 | Extreme positive floating exponents | -| nlohmann | 231 | 4 | Extreme positive floating exponents | -| JSON-C | 225 | 10 | Duplicate keys, NUL keys, native numeric ranges | -| Jansson | 225 | 10 | Duplicate keys, NUL keys, native numeric ranges | -| cJSON | 227 | 8 | Decoded NUL, native numeric ranges | -| JsonCpp | 233 | 2 | Duplicate keys | -| jsoncons | 235 | 0 | None | -| simdjson | 229 | 6 | Native integer/floating ranges | -| yyjson | 235 | 0 | None | +| YAJL | 242 | 0 | None | +| RapidJSON | 238 | 4 | Extreme positive floating exponents | +| nlohmann | 238 | 4 | Extreme positive floating exponents | +| JSON-C | 232 | 10 | Duplicate keys, NUL keys, native numeric ranges | +| Jansson | 232 | 10 | Duplicate keys, NUL keys, native numeric ranges | +| cJSON | 234 | 8 | Decoded NUL, native numeric ranges | +| JsonCpp | 240 | 2 | Duplicate keys | +| jsoncons | 242 | 0 | None | +| simdjson | 236 | 6 | Native integer/floating ranges | +| yyjson | 242 | 0 | None | Each failing valid case is tested in output and silent mode. Unsupported ranges produce diagnostics, with no silent integer rounding. Assertions are unchanged -for the required contract. There were **2339 passes, 44 failures, 0 skips**, plus +for the required contract. There were **2409 passes, 44 failures, 0 skips**, plus **2 separately checked numeric spelling differences**. This includes 33 exact -jsoncons/YAJL comparisons. `tests/run_tests.sh JSONCONS` returned 0 with **268 -passes, 0 failures, 0 skips, 2 differences**; the YAJL run returned 0 with **235 +jsoncons/YAJL comparisons. `tests/run_tests.sh JSONCONS` returned 0 with **275 +passes, 0 failures, 0 skips, 2 differences**; the YAJL run returned 0 with **242 passes**. No crash or timeout was reported. `make check` runs the six direct API/path harnesses. They contain 181 PathHandler, @@ -183,11 +186,43 @@ rejections for hidden reports, as a native rejection and a sanitizer can both exit nonzero. The sanitizer run used O1, frame pointers, non-PIE executables, leak detection, and all ten available engines. +## Sonar corrections and retained API advisories + +Automatic C-family analysis is configured to report recommendations for C++17, +matching the build, rather than facilities from later standards. No rules, +source files or required regression assertions are excluded. Shared test RAII +owns private fixture directories, descriptors, captured output and diagnostic +strings; a C-compiled helper releases C/POSIX diagnostics with their matching +allocator. The CLI validates iteration counts before allocation and bounds its +timing loop by the actual allocation size. YAJL's private state is grouped inline +without an extra allocation. Native validation and path helpers have simpler +control flow and shared initialization where behavior is identical. + +RapidJSON's fresh reader owns one bounded vector scratch arena for token bytes +and parser state. Reallocation preserves bytes and checks the budget and scalar +alignment; the allocator releases its storage when the reader is destroyed. +This follows RapidJSON's allocator lifetime rather than exposing raw ownership +to callers. The public C-compatible parser function types remain unchanged. + +Four const recommendations are intentionally retained after the user's explicit +compatibility decision: `rj_parse_buffer`, `rj_parse_file`, `rj_parser_cleanup` +and `nl_parser_cleanup`. Adding `const` to their parser pointer parameters would +change the exact function types and break callers that use existing typed +function pointers, even though direct calls and C symbol names would still work. +These findings are documented, not suppressed or marked accepted in Sonar. + +Verification uses the existing GitHub SonarQube Cloud checks for +`Easton97-Jens_jsonbench`. The environment cannot access the direct Sonar API, +so this does not certify a complete project-wide issue inventory. Cloud checks +report 0% coverage because these independently executed tests are not imported +as line coverage; 5,779 passing assertions do not establish a coverage percentage. + ## Measured jsoncons/YAJL comparison -The benchmark helper now attempts every requested YAJL input size. With five -fresh-process samples after one warmup per variant, the current binary and the -preserved original DOM executable produced: +The benchmark helper attempts every requested YAJL input size. These measurements +are from the integrated adapter snapshot `e6fe3a3`, before the subsequent Sonar +corrections. With five fresh-process samples after one warmup per variant, that +binary and the preserved original DOM executable produced: | Input | Original jsoncons DOM (ms / KiB RSS) | jsoncons SAX (ms / KiB RSS) | YAJL streaming (ms / KiB RSS) | | --- | --- | --- | --- | diff --git a/src/cjsonparser.c b/src/cjsonparser.c index d1b401e..abcc0b2 100644 --- a/src/cjsonparser.c +++ b/src/cjsonparser.c @@ -53,13 +53,9 @@ int cjson_parse_file(cjson_parser *parser, const char *filename, char **error_ms const char *end = NULL; cJSON *root = NULL; int rc; - if (error_msg == NULL) return -1; - *error_msg = NULL; - if (parser == NULL || filename == NULL) return dom_error(error_msg, "Invalid cJSON parser arguments"); - rc = dom_locale_begin(&locale, error_msg); + rc = dom_begin_numeric_file(parser, filename, "Invalid cJSON parser arguments", + &locale, &content, &length, error_msg); if (rc != 0) return rc; - rc = dom_read_file(filename, &content, &length, error_msg); - if (rc != 0) goto done; /* cJSON has no decoded string lengths and stores all numbers as doubles. * Reject NUL and unsafe integers instead of silently truncating/rounding. */ rc = dom_preflight_numbers(content, length, &parser->settings, &cjson_policy, diff --git a/src/domarguments.h b/src/domarguments.h index 5f484ce..0f504e3 100644 --- a/src/domarguments.h +++ b/src/domarguments.h @@ -139,6 +139,23 @@ static int dom_read_file(const char *filename, char **content, size_t *length, return 0; } +/* Numeric DOM libraries need the same argument validation, file input, and + * thread-locale lifetime. Native parsing and native resource cleanup stay in + * their adapters. A read failure restores the locale before returning. */ +static inline int dom_begin_numeric_file(const void *parser, const char *filename, + const char *invalid_message, dom_locale_guard *locale, + char **content, size_t *length, char **error) { + int result; + if (error == NULL) return -1; + *error = NULL; + if (parser == NULL || filename == NULL) return dom_error(error, invalid_message); + result = dom_locale_begin(locale, error); + if (result != 0) return result; + result = dom_read_file(filename, content, length, error); + if (result != 0) return dom_locale_finish(locale, result, error); + return 0; +} + static int dom_space(unsigned char c) { return c == ' ' || c == '\t' || c == '\r' || c == '\n'; } diff --git a/src/janssonparser.c b/src/janssonparser.c index b89d445..8918c38 100644 --- a/src/janssonparser.c +++ b/src/janssonparser.c @@ -64,13 +64,9 @@ int jansson_parse_file(jansson_parser *parser, const char *filename, char **erro json_error_t native_error; json_t *root = NULL; int rc; - if (error_msg == NULL) return -1; - *error_msg = NULL; - if (parser == NULL || filename == NULL) return dom_error(error_msg, "Invalid Jansson parser arguments"); - rc = dom_locale_begin(&locale, error_msg); + rc = dom_begin_numeric_file(parser, filename, "Invalid Jansson parser arguments", + &locale, &content, &length, error_msg); if (rc != 0) return rc; - rc = dom_read_file(filename, &content, &length, error_msg); - if (rc != 0) goto done; rc = dom_preflight_numbers(content, length, &parser->settings, &jansson_policy, &keys, &numbers, error_msg); if (rc != 0) goto done; diff --git a/src/jsoncparser.c b/src/jsoncparser.c index 7c98316..9b6624e 100644 --- a/src/jsoncparser.c +++ b/src/jsoncparser.c @@ -198,13 +198,9 @@ int jsonc_parse_file(jsonc_parser *parser, const char *filename, char **error_ms struct json_object *root = NULL; dom_settings check; int rc; - if (error_msg == NULL) return -1; - *error_msg = NULL; - if (parser == NULL || filename == NULL) return dom_error(error_msg, "Invalid JSON-C parser arguments"); - rc = dom_locale_begin(&locale, error_msg); + rc = dom_begin_numeric_file(parser, filename, "Invalid JSON-C parser arguments", + &locale, &content, &length, error_msg); if (rc != 0) return rc; - rc = dom_read_file(filename, &content, &length, error_msg); - if (rc != 0) goto done; rc = dom_preflight_numbers(content, length, &parser->settings, &jsonc_policy, &keys, &numbers, error_msg); if (rc != 0) goto done; diff --git a/tests/Makefile.am b/tests/Makefile.am index 2c8835b..6e098c0 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -8,19 +8,19 @@ AM_CXXFLAGS = -Wall -std=c++17 check_PROGRAMS = pathhandler_tests jsoncons_api_tests yajl_api_tests \ sax_adapter_api_tests dom_adapter_api_tests cpp_adapter_api_tests TESTS = $(check_PROGRAMS) -noinst_HEADERS = test_support.h +noinst_HEADERS = test_support.h test_c_memory.h -pathhandler_tests_SOURCES = pathhandler_tests.cpp +pathhandler_tests_SOURCES = pathhandler_tests.cpp test_c_memory.c -jsoncons_api_tests_SOURCES = jsoncons_api_tests.cpp ../src/jsonconsparser.cpp +jsoncons_api_tests_SOURCES = jsoncons_api_tests.cpp test_c_memory.c ../src/jsonconsparser.cpp jsoncons_api_tests_CPPFLAGS = $(AM_CPPFLAGS) -yajl_api_tests_SOURCES = yajl_api_tests.cpp ../src/yajlparser.c +yajl_api_tests_SOURCES = yajl_api_tests.cpp test_c_memory.c ../src/yajlparser.c yajl_api_tests_CPPFLAGS = $(AM_CPPFLAGS) $(YAJL_CFLAGS) yajl_api_tests_LDFLAGS = $(YAJL_LDFLAGS) yajl_api_tests_LDADD = $(YAJL_LDADD) -sax_adapter_api_tests_SOURCES = sax_adapter_api_tests.cpp \ +sax_adapter_api_tests_SOURCES = sax_adapter_api_tests.cpp test_c_memory.c \ ../src/rjparser.cpp ../src/nlparser.cpp sax_adapter_api_tests_CPPFLAGS = $(AM_CPPFLAGS) @@ -35,7 +35,7 @@ dom_adapter_api_tests_LDFLAGS = $(JANSSON_LDFLAGS) $(JSONC_LDFLAGS) \ dom_adapter_api_tests_LDADD = $(JANSSON_LDADD) $(JSONC_LDADD) \ $(CJSON_LDADD) $(YYJSON_LDADD) -lm -cpp_adapter_api_tests_SOURCES = cpp_adapter_api_tests.cpp \ +cpp_adapter_api_tests_SOURCES = cpp_adapter_api_tests.cpp test_c_memory.c \ ../src/jsoncppparser.cpp ../src/simdjsonparser.cpp cpp_adapter_api_tests_CPPFLAGS = $(AM_CPPFLAGS) $(JSONCPP_CFLAGS) $(SIMDJSON_CFLAGS) cpp_adapter_api_tests_LDFLAGS = $(JSONCPP_LDFLAGS) $(SIMDJSON_LDFLAGS) diff --git a/tests/README.md b/tests/README.md index 688906d..71ad8e2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -94,6 +94,12 @@ installed; that subcheck reports an explicit skip otherwise. The C++ harness preserves the simdjson safety ceiling regression: 512 containers accepted with an enlarged configured limit, 513/999/2000 rejected diagnostically. +The five C++ harnesses share `test_support.h` for private fixture directories, +file descriptors, captured output and diagnostic ownership. Their RAII deleter +calls the C-compiled `test_c_memory.c` bridge to release C/POSIX-allocated +diagnostics with the matching allocator. Automake builds and links this helper; +standalone C++ harness commands must also link its C object as shown above. + These API tests target the configured source tree, independently of a `JSONBENCH_BINARY` override. Missing engines return 77 and Automake reports SKIP; enabled compilation or test failures remain failures. CLI required tests From c72cb5a4a4550e9628737b142e34e38030ff1899 Mon Sep 17 00:00:00 2001 From: jens Steinbacher Date: Tue, 13 Oct 2026 18:09:04 +0000 Subject: [PATCH 16/18] Define stateless scratch allocator comparisons as hidden friends --- src/rjparser.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/rjparser.cpp b/src/rjparser.cpp index ebb1b5e..77ea531 100644 --- a/src/rjparser.cpp +++ b/src/rjparser.cpp @@ -65,8 +65,10 @@ class ScratchAllocator { void deallocate(T* data, std::size_t count) const noexcept { std::allocator{}.deallocate(data, count); } - template bool operator==(const ScratchAllocator&) const noexcept { return true; } - template bool operator!=(const ScratchAllocator&) const noexcept { return false; } + template + friend bool operator==(const ScratchAllocator&, const ScratchAllocator&) noexcept { return true; } + template + friend bool operator!=(const ScratchAllocator&, const ScratchAllocator&) noexcept { return false; } }; // Each parse here creates a fresh GenericReader with its own stack allocator. From 702b3aaa0421a4f5da1b8d32828dfd8894dfb3e5 Mon Sep 17 00:00:00 2001 From: jens Steinbacher Date: Tue, 13 Oct 2026 18:16:57 +0000 Subject: [PATCH 17/18] Share native C++ argument output and remove duplicate checks --- docs/adapter-stack.md | 5 +++++ src/Makefile.am | 1 + src/cpparguments.h | 36 ++++++++++++++++++++++++++++++++++++ src/jsoncppparser.cpp | 17 +++-------------- src/simdjsonparser.cpp | 17 +++-------------- 5 files changed, 48 insertions(+), 28 deletions(-) create mode 100644 src/cpparguments.h diff --git a/docs/adapter-stack.md b/docs/adapter-stack.md index a0e8b1d..eb7ef18 100644 --- a/docs/adapter-stack.md +++ b/docs/adapter-stack.md @@ -198,6 +198,11 @@ timing loop by the actual allocation size. YAJL's private state is grouped inlin without an extra allocation. Native validation and path helpers have simpler control flow and shared initialization where behavior is identical. +JsonCpp and simdjson share their identical scalar argument-emission helper: +length-aware path/value checks, argument counting, silence and checked output. +Their native parsers, traversal, depth ceilings and range checks remain in the +individual adapters. + RapidJSON's fresh reader owns one bounded vector scratch arena for token bytes and parser state. Reallocation preserves bytes and checks the budget and scalar alignment; the allocator releases its storage when the reader is destroyed. diff --git a/src/Makefile.am b/src/Makefile.am index 75d7f0b..f48eed1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -35,6 +35,7 @@ jsonbench_LDADD = $(YAJL_LDADD) $(JANSSON_LDADD) $(JSONC_LDADD) \ # Jansson must precede JSON-C for overlapping unversioned object APIs. # JSON-C traversal uses linkhash to avoid ambiguous iterator calls. noinst_HEADERS = cjsonparser.h \ + cpparguments.h \ domarguments.h \ glazeparser.h \ janssonparser.h \ diff --git a/src/cpparguments.h b/src/cpparguments.h new file mode 100644 index 0000000..5e2ad94 --- /dev/null +++ b/src/cpparguments.h @@ -0,0 +1,36 @@ +#ifndef JSONBENCH_CPP_ARGUMENTS_H +#define JSONBENCH_CPP_ARGUMENTS_H + +#include "pathhandler.h" + +#include +#include +#include +#include + +namespace jsonbench { + +// Both native C++ adapters share this argument contract. Keep validation and +// counting active in silent mode, and preserve binary values with sized writes. +// PathHandler::name() records size errors, so the path is intentionally mutable. +template +void emit_argument(unsigned int& count, unsigned int maximum, bool silent, + PathHandler& path, std::string_view value) { + const std::string name = path.name(); + if (path.m_error) throw std::length_error("Argument name too long"); + if (value.size() >= JSON_STRING_SIZE) + throw std::length_error("Argument value too long"); + if (count >= maximum) + throw std::length_error("Argument number limit exceeded"); + ++count; + if (silent) return; + if (std::fwrite(name.data(), 1, name.size(), stdout) != name.size() || + std::fwrite(": ", 1, 2, stdout) != 2 || + std::fwrite(value.data(), 1, value.size(), stdout) != value.size() || + std::fputc('\n', stdout) == EOF) + throw OutputError("Unable to write JSON argument output"); +} + +} // namespace jsonbench + +#endif diff --git a/src/jsoncppparser.cpp b/src/jsoncppparser.cpp index 6c2623d..7281b0a 100644 --- a/src/jsoncppparser.cpp +++ b/src/jsoncppparser.cpp @@ -2,7 +2,7 @@ #if HAVE_JSONCPP -#include "pathhandler.h" +#include "cpparguments.h" #include #include #include @@ -174,19 +174,8 @@ class Arguments { } void emit(const std::string& value) { - const std::string name = path_.name(); - check_path(); - if (value.size() >= JSON_STRING_SIZE) - throw std::length_error("Argument value too long"); - if (count_ >= settings_.max_args) - throw std::length_error("Argument number limit exceeded"); - ++count_; - if (settings_.silence) return; - if (std::fwrite(name.data(), 1, name.size(), stdout) != name.size() || - std::fwrite(": ", 1, 2, stdout) != 2 || - std::fwrite(value.data(), 1, value.size(), stdout) != value.size() || - std::fputc('\n', stdout) == EOF) - throw ParserError("Unable to write JSON argument output"); + jsonbench::emit_argument(count_, settings_.max_args, + settings_.silence, path_, value); } void visit_members(const Json::Value& value, unsigned int depth) { diff --git a/src/simdjsonparser.cpp b/src/simdjsonparser.cpp index 3ae8415..f94a59f 100644 --- a/src/simdjsonparser.cpp +++ b/src/simdjsonparser.cpp @@ -2,7 +2,7 @@ #if HAVE_SIMDJSON -#include "pathhandler.h" +#include "cpparguments.h" #include #include #include @@ -62,19 +62,8 @@ class Arguments { } void emit(std::string_view value) { - const std::string name = path_.name(); - check_path(); - if (value.size() >= JSON_STRING_SIZE) - throw std::length_error("Argument value too long"); - if (count_ >= settings_.max_args) - throw std::length_error("Argument number limit exceeded"); - ++count_; - if (settings_.silence) return; - if (std::fwrite(name.data(), 1, name.size(), stdout) != name.size() || - std::fwrite(": ", 1, 2, stdout) != 2 || - std::fwrite(value.data(), 1, value.size(), stdout) != value.size() || - std::fputc('\n', stdout) == EOF) - throw ParserError("Unable to write JSON argument output"); + jsonbench::emit_argument(count_, settings_.max_args, + settings_.silence, path_, value); } public: From f0627adaee5bd41f4cec8502d3264e7afc870c85 Mon Sep 17 00:00:00 2001 From: jens Steinbacher Date: Tue, 13 Oct 2026 18:20:18 +0000 Subject: [PATCH 18/18] Limit native parse diagnostic variable to its error check --- src/jsoncppparser.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jsoncppparser.cpp b/src/jsoncppparser.cpp index 7281b0a..88924fd 100644 --- a/src/jsoncppparser.cpp +++ b/src/jsoncppparser.cpp @@ -277,8 +277,8 @@ extern "C" int jsoncpp_parse_file(jsoncpp_parser *parser, const char *filename, std::min(parser->max_depth, 1023) + 1); std::unique_ptr reader(builder.newCharReader()); Json::Value root; - std::string errors; - if (!reader->parse(input.data(), input.data() + input.size(), &root, &errors)) + if (std::string errors; + !reader->parse(input.data(), input.data() + input.size(), &root, &errors)) throw std::invalid_argument(errors); Arguments(*parser, input).visit(root); if (!parser->silence && std::fflush(stdout) == EOF)